2024-11-13 18:11:59,261 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 18:11:59,284 main DEBUG Took 0.019955 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 18:11:59,285 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 18:11:59,285 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 18:11:59,286 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 18:11:59,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,308 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 18:11:59,325 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,334 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,336 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,337 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,338 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,342 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,342 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,343 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,343 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,344 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,345 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,345 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,346 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,347 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,351 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,352 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,353 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,354 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,354 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,355 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,356 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,356 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:11:59,360 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,360 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 18:11:59,366 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:11:59,368 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 18:11:59,370 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 18:11:59,371 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 18:11:59,373 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 18:11:59,373 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 18:11:59,384 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 18:11:59,386 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 18:11:59,389 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 18:11:59,389 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 18:11:59,389 main DEBUG createAppenders(={Console}) 2024-11-13 18:11:59,390 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-13 18:11:59,391 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 18:11:59,391 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-13 18:11:59,392 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 18:11:59,392 main DEBUG OutputStream closed 2024-11-13 18:11:59,393 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 18:11:59,393 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 18:11:59,393 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-13 18:11:59,494 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 18:11:59,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 18:11:59,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 18:11:59,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 18:11:59,500 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 18:11:59,500 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 18:11:59,501 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 18:11:59,501 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 18:11:59,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 18:11:59,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 18:11:59,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 18:11:59,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 18:11:59,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 18:11:59,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 18:11:59,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 18:11:59,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 18:11:59,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 18:11:59,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 18:11:59,508 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 18:11:59,509 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-13 18:11:59,509 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 18:11:59,510 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-13T18:11:59,879 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a 2024-11-13 18:11:59,883 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 18:11:59,884 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T18:11:59,896 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-13T18:11:59,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T18:11:59,943 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/cluster_2b3eda0b-9365-a4bb-f234-af4e8c2937bc, deleteOnExit=true 2024-11-13T18:11:59,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T18:11:59,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/test.cache.data in system properties and HBase conf 2024-11-13T18:11:59,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T18:11:59,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/hadoop.log.dir in system properties and HBase conf 2024-11-13T18:11:59,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T18:11:59,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T18:11:59,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T18:12:00,111 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T18:12:00,332 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:12:00,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T18:12:00,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T18:12:00,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T18:12:00,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T18:12:00,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T18:12:00,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T18:12:00,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T18:12:00,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T18:12:00,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T18:12:00,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/nfs.dump.dir in system properties and HBase conf 2024-11-13T18:12:00,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/java.io.tmpdir in system properties and HBase conf 2024-11-13T18:12:00,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T18:12:00,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T18:12:00,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T18:12:02,307 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T18:12:02,412 INFO [Time-limited test {}] log.Log(170): Logging initialized @4206ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T18:12:02,527 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:12:02,619 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:12:02,666 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:12:02,666 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:12:02,668 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T18:12:02,689 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:12:02,695 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72943464{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:12:02,696 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3061ce29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:12:03,056 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5af17d75{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/java.io.tmpdir/jetty-localhost-42261-hadoop-hdfs-3_4_1-tests_jar-_-any-12695391644770583073/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:12:03,066 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@51799591{HTTP/1.1, (http/1.1)}{localhost:42261} 2024-11-13T18:12:03,067 INFO [Time-limited test {}] server.Server(415): Started @4862ms 2024-11-13T18:12:03,892 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:12:03,904 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:12:03,909 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:12:03,909 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:12:03,910 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T18:12:03,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3e3aef44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:12:03,913 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b46b523{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:12:04,059 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6cc47bf8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/java.io.tmpdir/jetty-localhost-33055-hadoop-hdfs-3_4_1-tests_jar-_-any-2419901305084061597/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:12:04,060 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@381b1fb4{HTTP/1.1, (http/1.1)}{localhost:33055} 2024-11-13T18:12:04,061 INFO [Time-limited test {}] server.Server(415): Started @5856ms 2024-11-13T18:12:04,125 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T18:12:04,747 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/cluster_2b3eda0b-9365-a4bb-f234-af4e8c2937bc/data/data1/current/BP-225481686-172.17.0.3-1731521521390/current, will proceed with Du for space computation calculation, 2024-11-13T18:12:04,749 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/cluster_2b3eda0b-9365-a4bb-f234-af4e8c2937bc/data/data2/current/BP-225481686-172.17.0.3-1731521521390/current, will proceed with Du for space computation calculation, 2024-11-13T18:12:04,853 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T18:12:04,932 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb76451afa95be6a with lease ID 0x69cd9e256c0faac0: Processing first storage report for DS-67e46b83-0683-4d61-82a4-1eb61eebe7e9 from datanode DatanodeRegistration(127.0.0.1:38473, datanodeUuid=b9fb38ef-faf7-4350-ae02-268f0debde47, infoPort=35171, infoSecurePort=0, ipcPort=41559, storageInfo=lv=-57;cid=testClusterID;nsid=1261592615;c=1731521521390) 2024-11-13T18:12:04,935 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb76451afa95be6a with lease ID 0x69cd9e256c0faac0: from storage DS-67e46b83-0683-4d61-82a4-1eb61eebe7e9 node DatanodeRegistration(127.0.0.1:38473, datanodeUuid=b9fb38ef-faf7-4350-ae02-268f0debde47, infoPort=35171, infoSecurePort=0, ipcPort=41559, storageInfo=lv=-57;cid=testClusterID;nsid=1261592615;c=1731521521390), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-13T18:12:04,935 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb76451afa95be6a with lease ID 0x69cd9e256c0faac0: Processing first storage report for DS-affd5661-130c-4f3e-9f85-68705ed6bb5f from datanode DatanodeRegistration(127.0.0.1:38473, datanodeUuid=b9fb38ef-faf7-4350-ae02-268f0debde47, infoPort=35171, infoSecurePort=0, ipcPort=41559, storageInfo=lv=-57;cid=testClusterID;nsid=1261592615;c=1731521521390) 2024-11-13T18:12:04,936 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb76451afa95be6a with lease ID 0x69cd9e256c0faac0: from storage DS-affd5661-130c-4f3e-9f85-68705ed6bb5f node DatanodeRegistration(127.0.0.1:38473, datanodeUuid=b9fb38ef-faf7-4350-ae02-268f0debde47, infoPort=35171, infoSecurePort=0, ipcPort=41559, storageInfo=lv=-57;cid=testClusterID;nsid=1261592615;c=1731521521390), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T18:12:04,986 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a 2024-11-13T18:12:05,095 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/cluster_2b3eda0b-9365-a4bb-f234-af4e8c2937bc/zookeeper_0, clientPort=58684, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/cluster_2b3eda0b-9365-a4bb-f234-af4e8c2937bc/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/cluster_2b3eda0b-9365-a4bb-f234-af4e8c2937bc/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T18:12:05,108 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58684 2024-11-13T18:12:05,119 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:12:05,123 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:12:05,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741825_1001 (size=7) 2024-11-13T18:12:05,779 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1 with version=8 2024-11-13T18:12:05,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/hbase-staging 2024-11-13T18:12:05,884 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T18:12:06,218 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:12:06,233 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:12:06,233 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:12:06,240 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:12:06,240 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:12:06,240 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:12:06,424 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T18:12:06,492 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T18:12:06,505 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T18:12:06,510 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:12:06,547 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1025 (auto-detected) 2024-11-13T18:12:06,548 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T18:12:06,570 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37165 2024-11-13T18:12:06,599 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37165 connecting to ZooKeeper ensemble=127.0.0.1:58684 2024-11-13T18:12:06,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:371650x0, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:12:06,654 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37165-0x1003a57c7cd0000 connected 2024-11-13T18:12:06,700 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:12:06,706 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:12:06,719 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:12:06,724 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1, hbase.cluster.distributed=false 2024-11-13T18:12:06,759 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:12:06,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37165 2024-11-13T18:12:06,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37165 2024-11-13T18:12:06,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37165 2024-11-13T18:12:06,774 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37165 2024-11-13T18:12:06,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37165 2024-11-13T18:12:06,958 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:12:06,961 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:12:06,961 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:12:06,962 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:12:06,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:12:06,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:12:06,966 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:12:06,970 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:12:06,976 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44483 2024-11-13T18:12:06,979 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44483 connecting to ZooKeeper ensemble=127.0.0.1:58684 2024-11-13T18:12:06,982 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:12:06,987 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:12:07,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444830x0, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:12:07,021 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44483-0x1003a57c7cd0001 connected 2024-11-13T18:12:07,022 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:12:07,028 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:12:07,044 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:12:07,049 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:12:07,069 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:12:07,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44483 2024-11-13T18:12:07,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44483 2024-11-13T18:12:07,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44483 2024-11-13T18:12:07,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44483 2024-11-13T18:12:07,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44483 2024-11-13T18:12:07,120 DEBUG [M:0;fda96cf025dc:37165 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fda96cf025dc:37165 2024-11-13T18:12:07,129 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fda96cf025dc,37165,1731521525941 2024-11-13T18:12:07,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:12:07,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:12:07,141 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fda96cf025dc,37165,1731521525941 2024-11-13T18:12:07,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:07,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:12:07,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:07,178 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T18:12:07,179 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fda96cf025dc,37165,1731521525941 from backup master directory 2024-11-13T18:12:07,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:12:07,184 WARN [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:12:07,184 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fda96cf025dc,37165,1731521525941 2024-11-13T18:12:07,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fda96cf025dc,37165,1731521525941 2024-11-13T18:12:07,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:12:07,186 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T18:12:07,188 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T18:12:07,278 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/hbase.id] with ID: c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab 2024-11-13T18:12:07,279 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/.tmp/hbase.id 2024-11-13T18:12:07,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741826_1002 (size=42) 2024-11-13T18:12:07,299 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/.tmp/hbase.id]:[hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/hbase.id] 2024-11-13T18:12:07,364 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:12:07,380 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T18:12:07,418 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 35ms. 2024-11-13T18:12:07,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:07,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:07,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741827_1003 (size=196) 2024-11-13T18:12:07,501 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:12:07,503 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T18:12:07,532 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:07,538 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:12:07,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741828_1004 (size=1189) 2024-11-13T18:12:07,617 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store 2024-11-13T18:12:07,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741829_1005 (size=34) 2024-11-13T18:12:07,656 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T18:12:07,661 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:12:07,662 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:12:07,663 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:12:07,663 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:12:07,665 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:12:07,665 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:12:07,665 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:12:07,667 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731521527662Disabling compacts and flushes for region at 1731521527662Disabling writes for close at 1731521527665 (+3 ms)Writing region close event to WAL at 1731521527665Closed at 1731521527665 2024-11-13T18:12:07,670 WARN [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/.initializing 2024-11-13T18:12:07,670 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/WALs/fda96cf025dc,37165,1731521525941 2024-11-13T18:12:07,682 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:12:07,703 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C37165%2C1731521525941, suffix=, logDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/WALs/fda96cf025dc,37165,1731521525941, archiveDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/oldWALs, maxLogs=10 2024-11-13T18:12:07,731 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/WALs/fda96cf025dc,37165,1731521525941/fda96cf025dc%2C37165%2C1731521525941.1731521527708, exclude list is [], retry=0 2024-11-13T18:12:07,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38473,DS-67e46b83-0683-4d61-82a4-1eb61eebe7e9,DISK] 2024-11-13T18:12:07,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T18:12:07,815 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/WALs/fda96cf025dc,37165,1731521525941/fda96cf025dc%2C37165%2C1731521525941.1731521527708 2024-11-13T18:12:07,817 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35171:35171)] 2024-11-13T18:12:07,817 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:12:07,818 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:12:07,822 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,823 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,869 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,905 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T18:12:07,910 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:07,913 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:12:07,914 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,922 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T18:12:07,922 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:07,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:07,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T18:12:07,928 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:07,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:07,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T18:12:07,933 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:07,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:07,934 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,939 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,941 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,950 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,951 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,955 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:12:07,959 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:12:07,964 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:12:07,966 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62686700, jitterRate=-0.06589537858963013}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:12:07,971 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731521527838Initializing all the Stores at 1731521527841 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521527842 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521527843 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521527843Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521527843Cleaning up temporary data from old regions at 1731521527951 (+108 ms)Region opened successfully at 1731521527971 (+20 ms) 2024-11-13T18:12:07,972 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T18:12:08,008 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f756865, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:12:08,044 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T18:12:08,058 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T18:12:08,058 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T18:12:08,062 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:12:08,063 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T18:12:08,069 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T18:12:08,069 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T18:12:08,107 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T18:12:08,119 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T18:12:08,122 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T18:12:08,126 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T18:12:08,129 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T18:12:08,131 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T18:12:08,133 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T18:12:08,138 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T18:12:08,140 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T18:12:08,142 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T18:12:08,145 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T18:12:08,170 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T18:12:08,172 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T18:12:08,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:12:08,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:12:08,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:08,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:08,180 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fda96cf025dc,37165,1731521525941, sessionid=0x1003a57c7cd0000, setting cluster-up flag (Was=false) 2024-11-13T18:12:08,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:08,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:08,200 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T18:12:08,203 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda96cf025dc,37165,1731521525941 2024-11-13T18:12:08,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:08,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:08,217 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T18:12:08,219 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda96cf025dc,37165,1731521525941 2024-11-13T18:12:08,227 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T18:12:08,299 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(746): ClusterId : c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab 2024-11-13T18:12:08,302 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:12:08,310 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:12:08,310 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:12:08,315 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:12:08,316 DEBUG [RS:0;fda96cf025dc:44483 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be2fafc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:12:08,317 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:12:08,330 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T18:12:08,337 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fda96cf025dc:44483 2024-11-13T18:12:08,339 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T18:12:08,341 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:12:08,342 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:12:08,342 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:12:08,345 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,37165,1731521525941 with port=44483, startcode=1731521526892 2024-11-13T18:12:08,346 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fda96cf025dc,37165,1731521525941 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T18:12:08,355 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:12:08,355 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:12:08,356 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:12:08,356 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:12:08,356 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fda96cf025dc:0, corePoolSize=10, maxPoolSize=10 2024-11-13T18:12:08,357 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,357 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:12:08,357 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,359 DEBUG [RS:0;fda96cf025dc:44483 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:12:08,367 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521558367 2024-11-13T18:12:08,367 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:12:08,368 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T18:12:08,369 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T18:12:08,370 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T18:12:08,375 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T18:12:08,375 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T18:12:08,376 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T18:12:08,376 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T18:12:08,376 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:08,378 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T18:12:08,383 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,388 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T18:12:08,396 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T18:12:08,397 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T18:12:08,403 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T18:12:08,403 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T18:12:08,405 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731521528405,5,FailOnTimeoutGroup] 2024-11-13T18:12:08,406 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731521528406,5,FailOnTimeoutGroup] 2024-11-13T18:12:08,406 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,407 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T18:12:08,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741831_1007 (size=1321) 2024-11-13T18:12:08,412 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,412 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,414 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T18:12:08,415 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1 2024-11-13T18:12:08,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741832_1008 (size=32) 2024-11-13T18:12:08,431 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54223, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:12:08,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:12:08,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:12:08,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:12:08,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:08,439 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37165 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,44483,1731521526892 2024-11-13T18:12:08,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:12:08,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:12:08,442 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37165 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:08,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:12:08,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:08,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:12:08,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:12:08,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:12:08,447 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:08,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:12:08,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:12:08,450 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:12:08,450 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:08,451 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:12:08,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:12:08,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740 2024-11-13T18:12:08,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740 2024-11-13T18:12:08,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:12:08,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:12:08,458 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:12:08,458 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1 2024-11-13T18:12:08,459 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45039 2024-11-13T18:12:08,459 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:12:08,461 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:12:08,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:12:08,466 DEBUG [RS:0;fda96cf025dc:44483 {}] zookeeper.ZKUtil(111): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,44483,1731521526892 2024-11-13T18:12:08,466 WARN [RS:0;fda96cf025dc:44483 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:12:08,467 INFO [RS:0;fda96cf025dc:44483 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:12:08,467 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:12:08,467 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/WALs/fda96cf025dc,44483,1731521526892 2024-11-13T18:12:08,468 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67792229, jitterRate=0.010182932019233704}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:12:08,472 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731521528432Initializing all the Stores at 1731521528435 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521528435Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521528435Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521528435Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521528435Cleaning up temporary data from old regions at 1731521528457 (+22 ms)Region opened successfully at 1731521528472 (+15 ms) 2024-11-13T18:12:08,472 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:12:08,472 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:12:08,472 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:12:08,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:12:08,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:12:08,474 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,44483,1731521526892] 2024-11-13T18:12:08,481 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:12:08,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731521528472Disabling compacts and flushes for region at 1731521528472Disabling writes for close at 1731521528473 (+1 ms)Writing region close event to WAL at 1731521528480 (+7 ms)Closed at 1731521528481 (+1 ms) 2024-11-13T18:12:08,485 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:12:08,486 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T18:12:08,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T18:12:08,505 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:12:08,507 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:12:08,508 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T18:12:08,529 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:12:08,537 INFO [RS:0;fda96cf025dc:44483 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:12:08,538 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,539 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:12:08,547 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:12:08,549 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,549 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,550 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,550 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,550 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,550 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,551 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:12:08,551 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,551 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,551 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,552 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,552 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,552 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:12:08,552 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:12:08,553 DEBUG [RS:0;fda96cf025dc:44483 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:12:08,560 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,561 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,561 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,561 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,561 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,561 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,44483,1731521526892-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:12:08,602 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:12:08,605 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,44483,1731521526892-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,606 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,606 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.Replication(171): fda96cf025dc,44483,1731521526892 started 2024-11-13T18:12:08,636 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:08,636 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,44483,1731521526892, RpcServer on fda96cf025dc/172.17.0.3:44483, sessionid=0x1003a57c7cd0001 2024-11-13T18:12:08,637 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:12:08,638 DEBUG [RS:0;fda96cf025dc:44483 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,44483,1731521526892 2024-11-13T18:12:08,638 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,44483,1731521526892' 2024-11-13T18:12:08,638 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:12:08,640 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:12:08,641 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:12:08,641 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:12:08,642 DEBUG [RS:0;fda96cf025dc:44483 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,44483,1731521526892 2024-11-13T18:12:08,642 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,44483,1731521526892' 2024-11-13T18:12:08,642 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:12:08,643 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:12:08,644 DEBUG [RS:0;fda96cf025dc:44483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:12:08,644 INFO [RS:0;fda96cf025dc:44483 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:12:08,644 INFO [RS:0;fda96cf025dc:44483 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:12:08,659 WARN [fda96cf025dc:37165 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T18:12:08,750 INFO [RS:0;fda96cf025dc:44483 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:12:08,753 INFO [RS:0;fda96cf025dc:44483 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C44483%2C1731521526892, suffix=, logDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/WALs/fda96cf025dc,44483,1731521526892, archiveDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/oldWALs, maxLogs=32 2024-11-13T18:12:08,771 DEBUG [RS:0;fda96cf025dc:44483 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/WALs/fda96cf025dc,44483,1731521526892/fda96cf025dc%2C44483%2C1731521526892.1731521528756, exclude list is [], retry=0 2024-11-13T18:12:08,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38473,DS-67e46b83-0683-4d61-82a4-1eb61eebe7e9,DISK] 2024-11-13T18:12:08,780 INFO [RS:0;fda96cf025dc:44483 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/WALs/fda96cf025dc,44483,1731521526892/fda96cf025dc%2C44483%2C1731521526892.1731521528756 2024-11-13T18:12:08,781 DEBUG [RS:0;fda96cf025dc:44483 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35171:35171)] 2024-11-13T18:12:08,912 DEBUG [fda96cf025dc:37165 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T18:12:08,925 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:08,931 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,44483,1731521526892, state=OPENING 2024-11-13T18:12:08,937 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T18:12:08,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:08,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:12:08,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:12:08,942 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:12:08,943 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:12:08,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:12:09,126 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:12:09,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37713, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:12:09,145 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T18:12:09,145 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:12:09,146 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T18:12:09,150 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C44483%2C1731521526892.meta, suffix=.meta, logDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/WALs/fda96cf025dc,44483,1731521526892, archiveDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/oldWALs, maxLogs=32 2024-11-13T18:12:09,174 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/WALs/fda96cf025dc,44483,1731521526892/fda96cf025dc%2C44483%2C1731521526892.meta.1731521529151.meta, exclude list is [], retry=0 2024-11-13T18:12:09,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38473,DS-67e46b83-0683-4d61-82a4-1eb61eebe7e9,DISK] 2024-11-13T18:12:09,184 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/WALs/fda96cf025dc,44483,1731521526892/fda96cf025dc%2C44483%2C1731521526892.meta.1731521529151.meta 2024-11-13T18:12:09,185 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35171:35171)] 2024-11-13T18:12:09,185 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:12:09,187 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T18:12:09,191 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T18:12:09,197 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T18:12:09,202 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T18:12:09,202 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:12:09,202 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T18:12:09,203 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T18:12:09,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:12:09,210 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:12:09,210 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:09,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:12:09,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:12:09,213 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:12:09,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:09,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:12:09,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:12:09,217 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:12:09,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:09,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:12:09,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:12:09,221 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:12:09,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:09,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:12:09,223 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:12:09,225 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740 2024-11-13T18:12:09,228 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740 2024-11-13T18:12:09,232 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:12:09,232 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:12:09,233 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:12:09,239 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:12:09,241 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61729089, jitterRate=-0.08016489446163177}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:12:09,241 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T18:12:09,244 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731521529203Writing region info on filesystem at 1731521529203Initializing all the Stores at 1731521529207 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521529207Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521529207Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521529207Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521529207Cleaning up temporary data from old regions at 1731521529232 (+25 ms)Running coprocessor post-open hooks at 1731521529241 (+9 ms)Region opened successfully at 1731521529244 (+3 ms) 2024-11-13T18:12:09,255 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731521529115 2024-11-13T18:12:09,270 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T18:12:09,271 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T18:12:09,273 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:09,276 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,44483,1731521526892, state=OPEN 2024-11-13T18:12:09,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:12:09,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:12:09,284 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:12:09,284 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:12:09,285 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:09,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T18:12:09,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,44483,1731521526892 in 340 msec 2024-11-13T18:12:09,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T18:12:09,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 800 msec 2024-11-13T18:12:09,302 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:12:09,302 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T18:12:09,329 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:09,331 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:09,355 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:09,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49415, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:09,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1180 sec 2024-11-13T18:12:09,381 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731521529380, completionTime=-1 2024-11-13T18:12:09,384 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T18:12:09,384 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:12:09,417 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-13T18:12:09,417 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731521589417 2024-11-13T18:12:09,417 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731521649417 2024-11-13T18:12:09,417 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-13T18:12:09,419 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,37165,1731521525941-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:09,419 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,37165,1731521525941-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:09,420 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,37165,1731521525941-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:09,421 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fda96cf025dc:37165, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:09,422 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:09,422 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:09,430 DEBUG [master/fda96cf025dc:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T18:12:09,459 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.274sec 2024-11-13T18:12:09,460 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T18:12:09,461 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T18:12:09,462 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T18:12:09,463 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T18:12:09,463 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T18:12:09,464 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,37165,1731521525941-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:12:09,464 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,37165,1731521525941-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T18:12:09,475 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T18:12:09,476 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T18:12:09,476 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,37165,1731521525941-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:12:09,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dae98d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:09,515 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T18:12:09,515 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T18:12:09,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:09,524 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:09,534 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6c304e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:09,547 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:09,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b7aae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:09,551 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:09,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:09,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fda96cf025dc,37165,1731521525941 2024-11-13T18:12:09,594 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=660, ProcessCount=11, AvailableMemoryMB=260 2024-11-13T18:12:09,603 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:12:09,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,37165,1731521525941 2024-11-13T18:12:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@e3ff8cb 2024-11-13T18:12:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:12:09,615 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:12:09,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:12:09,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:12:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:12:09,637 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:12:09,640 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-13T18:12:09,640 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:09,643 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:12:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:12:09,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741835_1011 (size=1061) 2024-11-13T18:12:09,663 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1 2024-11-13T18:12:09,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741836_1012 (size=53) 2024-11-13T18:12:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:12:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:12:10,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:12:10,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 401e2f643e0c1bcf64dcec1f711ca8ed, disabling compactions & flushes 2024-11-13T18:12:10,075 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:10,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:10,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. after waiting 0 ms 2024-11-13T18:12:10,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:10,075 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:10,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 401e2f643e0c1bcf64dcec1f711ca8ed: Waiting for close lock at 1731521530075Disabling compacts and flushes for region at 1731521530075Disabling writes for close at 1731521530075Writing region close event to WAL at 1731521530075Closed at 1731521530075 2024-11-13T18:12:10,078 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:12:10,084 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731521530078"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521530078"}]},"ts":"1731521530078"} 2024-11-13T18:12:10,088 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:12:10,091 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:12:10,094 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521530091"}]},"ts":"1731521530091"} 2024-11-13T18:12:10,099 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:12:10,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=401e2f643e0c1bcf64dcec1f711ca8ed, ASSIGN}] 2024-11-13T18:12:10,104 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=401e2f643e0c1bcf64dcec1f711ca8ed, ASSIGN 2024-11-13T18:12:10,106 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=401e2f643e0c1bcf64dcec1f711ca8ed, ASSIGN; state=OFFLINE, location=fda96cf025dc,44483,1731521526892; forceNewPlan=false, retain=false 2024-11-13T18:12:10,257 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=401e2f643e0c1bcf64dcec1f711ca8ed, regionState=OPENING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:10,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=401e2f643e0c1bcf64dcec1f711ca8ed, ASSIGN because future has completed 2024-11-13T18:12:10,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:12:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:12:10,424 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:10,424 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:12:10,425 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,425 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:12:10,425 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,425 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,428 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,432 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:12:10,433 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 401e2f643e0c1bcf64dcec1f711ca8ed columnFamilyName A 2024-11-13T18:12:10,433 DEBUG [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:10,434 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] regionserver.HStore(327): Store=401e2f643e0c1bcf64dcec1f711ca8ed/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:10,434 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,436 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:12:10,437 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 401e2f643e0c1bcf64dcec1f711ca8ed columnFamilyName B 2024-11-13T18:12:10,437 DEBUG [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:10,438 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] regionserver.HStore(327): Store=401e2f643e0c1bcf64dcec1f711ca8ed/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:10,438 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,440 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:12:10,441 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 401e2f643e0c1bcf64dcec1f711ca8ed columnFamilyName C 2024-11-13T18:12:10,441 DEBUG [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:10,442 INFO [StoreOpener-401e2f643e0c1bcf64dcec1f711ca8ed-1 {}] regionserver.HStore(327): Store=401e2f643e0c1bcf64dcec1f711ca8ed/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:10,442 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:10,442 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,444 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,444 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,446 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,446 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,447 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:12:10,449 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,453 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:12:10,454 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 401e2f643e0c1bcf64dcec1f711ca8ed; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61402556, jitterRate=-0.08503061532974243}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:12:10,454 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:10,455 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 401e2f643e0c1bcf64dcec1f711ca8ed: Running coprocessor pre-open hook at 1731521530426Writing region info on filesystem at 1731521530426Initializing all the Stores at 1731521530428 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521530428Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521530428Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521530428Cleaning up temporary data from old regions at 1731521530446 (+18 ms)Running coprocessor post-open hooks at 1731521530454 (+8 ms)Region opened successfully at 1731521530455 (+1 ms) 2024-11-13T18:12:10,458 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., pid=6, masterSystemTime=1731521530417 2024-11-13T18:12:10,461 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:10,461 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:10,462 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=401e2f643e0c1bcf64dcec1f711ca8ed, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:10,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:12:10,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T18:12:10,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 in 205 msec 2024-11-13T18:12:10,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T18:12:10,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=401e2f643e0c1bcf64dcec1f711ca8ed, ASSIGN in 371 msec 2024-11-13T18:12:10,481 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:12:10,481 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521530481"}]},"ts":"1731521530481"} 2024-11-13T18:12:10,485 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:12:10,487 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:12:10,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 860 msec 2024-11-13T18:12:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:12:10,786 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:10,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42fb40b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:10,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:10,795 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253618fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:10,798 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dec8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,800 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:10,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:10,807 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:12:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,37165,1731521525941 2024-11-13T18:12:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1df1000 2024-11-13T18:12:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:12:10,813 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:12:10,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64a6f7bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:10,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:10,818 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:10,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:10,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:10,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21add1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:10,821 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@313a86b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:10,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:10,825 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:10,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86d3f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:10,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:10,832 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:10,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:10,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:10,836 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f9ce8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:10,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:10,839 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:10,842 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:10,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e0b13b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6751f3e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:10,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:10,845 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db48f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:10,848 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c0c6470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@486467a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:10,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:10,852 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:10,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:10,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:10,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@537574c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:10,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:10,855 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18101d81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fce0b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:10,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:10,857 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74deee7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:10,860 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:10,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e021fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc519b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:10,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:10,864 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@905b4d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:10,867 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526369d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f78bb86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:10,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:10,869 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ce558b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:10,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:10,871 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280a2bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:10,878 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:10,879 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:10,879 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:10,879 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:10,880 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:10,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:10,881 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:10,881 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:10,881 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:10,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:10,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:10,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:10,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:10,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:10,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:10,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:10,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:10,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:12:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:12:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-13T18:12:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:12:10,925 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:12:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:12:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:12:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:12:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:12:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:12:10,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:10,936 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:10,941 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:12:10,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:11,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:12:11,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:11,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:11,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:11,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:11,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:11,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:12:11,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/de7aa9cfcad543e782f24ef75477c7b8 is 50, key is test_row_0/A:col10/1731521531000/Put/seqid=0 2024-11-13T18:12:11,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:12:11,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:11,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741837_1013 (size=14341) 2024-11-13T18:12:11,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521591143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18724, totalHeapSize=42776, min timestamp=1731521531064, max timestamp=1731521531162Num uniques 859; 2024-11-13T18:12:11,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=18724, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-13T18:12:11,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18724, totalHeapSize=42776, min timestamp=1731521531064, max timestamp=1731521531162Num uniques 859; 2024-11-13T18:12:11,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=18724, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-13T18:12:11,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18724, totalHeapSize=42776, min timestamp=1731521531064, max timestamp=1731521531162Num uniques 859; 2024-11-13T18:12:11,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=18724, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-13T18:12:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:12:11,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521591250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521591251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521591256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521591256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:12:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521591325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521591375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521591376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521591384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521591385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:12:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:11,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/de7aa9cfcad543e782f24ef75477c7b8 2024-11-13T18:12:11,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521591546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:12:11,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521591587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521591588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521591594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521591606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:12:11,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:11,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/bd75442033e646ee9fa21e2af11b096d is 50, key is test_row_0/B:col10/1731521531000/Put/seqid=0 2024-11-13T18:12:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741838_1014 (size=12001) 2024-11-13T18:12:11,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/bd75442033e646ee9fa21e2af11b096d 2024-11-13T18:12:11,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/bb2dd0ecb35d46b79cb8e20f8078f49e is 50, key is test_row_0/C:col10/1731521531000/Put/seqid=0 2024-11-13T18:12:11,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741839_1015 (size=12001) 2024-11-13T18:12:11,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:12:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521591864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521591894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521591905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521591914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521591915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:11,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:11,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:12:11,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:11,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:11,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:12:12,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:12:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:12,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:12,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:12,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:12,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/bb2dd0ecb35d46b79cb8e20f8078f49e 2024-11-13T18:12:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:12,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/de7aa9cfcad543e782f24ef75477c7b8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/de7aa9cfcad543e782f24ef75477c7b8 2024-11-13T18:12:12,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/de7aa9cfcad543e782f24ef75477c7b8, entries=200, sequenceid=13, filesize=14.0 K 2024-11-13T18:12:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/bd75442033e646ee9fa21e2af11b096d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd75442033e646ee9fa21e2af11b096d 2024-11-13T18:12:12,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd75442033e646ee9fa21e2af11b096d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:12:12,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/bb2dd0ecb35d46b79cb8e20f8078f49e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/bb2dd0ecb35d46b79cb8e20f8078f49e 2024-11-13T18:12:12,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/bb2dd0ecb35d46b79cb8e20f8078f49e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:12:12,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 401e2f643e0c1bcf64dcec1f711ca8ed in 1167ms, sequenceid=13, compaction requested=false 2024-11-13T18:12:12,181 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T18:12:12,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:12,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:12:12,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:12,262 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=127.47 KB heapSize=341.63 KB 2024-11-13T18:12:12,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:12,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f35dd5ef95344e888e00731538499a89 is 50, key is test_row_0/A:col10/1731521531249/Put/seqid=0 2024-11-13T18:12:12,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741840_1016 (size=12001) 2024-11-13T18:12:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:12,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:12,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521592445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521592450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521592457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521592463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521592490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521592595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521592596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521592635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521592635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521592636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,700 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f35dd5ef95344e888e00731538499a89 2024-11-13T18:12:12,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/0d9873147ee2455abb8d1a6435da5d4f is 50, key is test_row_0/B:col10/1731521531249/Put/seqid=0 2024-11-13T18:12:12,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741841_1017 (size=12001) 2024-11-13T18:12:12,744 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/0d9873147ee2455abb8d1a6435da5d4f 2024-11-13T18:12:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/c17f29ed65fa49f0a2bb592885a47a95 is 50, key is test_row_0/C:col10/1731521531249/Put/seqid=0 2024-11-13T18:12:12,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741842_1018 (size=12001) 2024-11-13T18:12:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521592816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521592827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521592876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521592876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:12,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521592885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:12:13,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521593154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521593165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,183 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/c17f29ed65fa49f0a2bb592885a47a95 2024-11-13T18:12:13,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f35dd5ef95344e888e00731538499a89 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f35dd5ef95344e888e00731538499a89 2024-11-13T18:12:13,209 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f35dd5ef95344e888e00731538499a89, entries=150, sequenceid=44, filesize=11.7 K 2024-11-13T18:12:13,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/0d9873147ee2455abb8d1a6435da5d4f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0d9873147ee2455abb8d1a6435da5d4f 2024-11-13T18:12:13,222 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0d9873147ee2455abb8d1a6435da5d4f, entries=150, sequenceid=44, filesize=11.7 K 2024-11-13T18:12:13,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/c17f29ed65fa49f0a2bb592885a47a95 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/c17f29ed65fa49f0a2bb592885a47a95 2024-11-13T18:12:13,240 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/c17f29ed65fa49f0a2bb592885a47a95, entries=150, sequenceid=44, filesize=11.7 K 2024-11-13T18:12:13,243 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~341.58 KB/349776, currentSize=60.38 KB/61830 for 401e2f643e0c1bcf64dcec1f711ca8ed in 981ms, sequenceid=44, compaction requested=false 2024-11-13T18:12:13,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:13,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:13,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T18:12:13,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T18:12:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:13,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T18:12:13,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3050 sec 2024-11-13T18:12:13,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T18:12:13,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:13,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:13,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:13,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:13,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:13,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:13,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f6ade35c4abf4794ac3c1879ecbd3796 is 50, key is test_row_0/A:col10/1731521533252/Put/seqid=0 2024-11-13T18:12:13,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3780 sec 2024-11-13T18:12:13,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741843_1019 (size=12001) 2024-11-13T18:12:13,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521593341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521593342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521593345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521593465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521593465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521593466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521593674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521593675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521593675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521593676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f6ade35c4abf4794ac3c1879ecbd3796 2024-11-13T18:12:13,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521593685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/33c724ff4278495dafded55d03e25f81 is 50, key is test_row_0/B:col10/1731521533252/Put/seqid=0 2024-11-13T18:12:13,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741844_1020 (size=12001) 2024-11-13T18:12:13,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521593985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521593985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521593985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:13,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:13,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/33c724ff4278495dafded55d03e25f81 2024-11-13T18:12:14,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/b2b687d651324c5e8df4ce2aa9a401af is 50, key is test_row_0/C:col10/1731521533252/Put/seqid=0 2024-11-13T18:12:14,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741845_1021 (size=12001) 2024-11-13T18:12:14,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/b2b687d651324c5e8df4ce2aa9a401af 2024-11-13T18:12:14,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f6ade35c4abf4794ac3c1879ecbd3796 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f6ade35c4abf4794ac3c1879ecbd3796 2024-11-13T18:12:14,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f6ade35c4abf4794ac3c1879ecbd3796, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T18:12:14,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/33c724ff4278495dafded55d03e25f81 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/33c724ff4278495dafded55d03e25f81 2024-11-13T18:12:14,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/33c724ff4278495dafded55d03e25f81, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T18:12:14,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/b2b687d651324c5e8df4ce2aa9a401af as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/b2b687d651324c5e8df4ce2aa9a401af 2024-11-13T18:12:14,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/b2b687d651324c5e8df4ce2aa9a401af, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T18:12:14,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 401e2f643e0c1bcf64dcec1f711ca8ed in 971ms, sequenceid=59, compaction requested=true 2024-11-13T18:12:14,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:14,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:14,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:14,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:14,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:14,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:14,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:14,228 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:14,229 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:14,232 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:14,232 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:14,234 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:14,234 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:14,235 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:14,235 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:14,235 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd75442033e646ee9fa21e2af11b096d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0d9873147ee2455abb8d1a6435da5d4f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/33c724ff4278495dafded55d03e25f81] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=35.2 K 2024-11-13T18:12:14,235 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/de7aa9cfcad543e782f24ef75477c7b8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f35dd5ef95344e888e00731538499a89, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f6ade35c4abf4794ac3c1879ecbd3796] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=37.4 K 2024-11-13T18:12:14,236 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting bd75442033e646ee9fa21e2af11b096d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731521530998 2024-11-13T18:12:14,236 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting de7aa9cfcad543e782f24ef75477c7b8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731521530989 2024-11-13T18:12:14,237 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d9873147ee2455abb8d1a6435da5d4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731521531243 2024-11-13T18:12:14,237 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f35dd5ef95344e888e00731538499a89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731521531243 2024-11-13T18:12:14,238 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6ade35c4abf4794ac3c1879ecbd3796, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521532434 2024-11-13T18:12:14,238 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c724ff4278495dafded55d03e25f81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521532434 2024-11-13T18:12:14,280 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#9 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:14,282 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/6f950db1699a4f589f5615b4a2e988df is 50, key is test_row_0/B:col10/1731521533252/Put/seqid=0 2024-11-13T18:12:14,303 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:14,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741846_1022 (size=12104) 2024-11-13T18:12:14,307 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/bf1f660d96f949e3b29504c58bd374b1 is 50, key is test_row_0/A:col10/1731521533252/Put/seqid=0 2024-11-13T18:12:14,322 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/6f950db1699a4f589f5615b4a2e988df as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6f950db1699a4f589f5615b4a2e988df 2024-11-13T18:12:14,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741847_1023 (size=12104) 2024-11-13T18:12:14,341 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/bf1f660d96f949e3b29504c58bd374b1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/bf1f660d96f949e3b29504c58bd374b1 2024-11-13T18:12:14,349 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into 6f950db1699a4f589f5615b4a2e988df(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:14,349 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:14,352 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=13, startTime=1731521534228; duration=0sec 2024-11-13T18:12:14,353 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:14,353 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:14,353 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:14,354 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into bf1f660d96f949e3b29504c58bd374b1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:14,354 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:14,354 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=13, startTime=1731521534225; duration=0sec 2024-11-13T18:12:14,355 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:14,355 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:14,355 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:14,355 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:14,355 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:14,356 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/bb2dd0ecb35d46b79cb8e20f8078f49e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/c17f29ed65fa49f0a2bb592885a47a95, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/b2b687d651324c5e8df4ce2aa9a401af] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=35.2 K 2024-11-13T18:12:14,356 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting bb2dd0ecb35d46b79cb8e20f8078f49e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731521530998 2024-11-13T18:12:14,357 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c17f29ed65fa49f0a2bb592885a47a95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731521531243 2024-11-13T18:12:14,358 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b2b687d651324c5e8df4ce2aa9a401af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521532434 2024-11-13T18:12:14,384 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#11 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:14,385 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/cf122331869145f79fe413b15910c473 is 50, key is test_row_0/C:col10/1731521533252/Put/seqid=0 2024-11-13T18:12:14,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741848_1024 (size=12104) 2024-11-13T18:12:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:14,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:12:14,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:14,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:14,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:14,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:14,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:14,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:14,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/c702b6df845548609f63775d4566fdbf is 50, key is test_row_0/A:col10/1731521533341/Put/seqid=0 2024-11-13T18:12:14,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741849_1025 (size=12001) 2024-11-13T18:12:14,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521594538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521594553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521594554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521594664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521594665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521594665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521594684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,690 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_1' in 401e2f643e0c1bcf64dcec1f711ca8ed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,712 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:12:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521594714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,720 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_2' in 401e2f643e0c1bcf64dcec1f711ca8ed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2275 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,773 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T18:12:14,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/cf122331869145f79fe413b15910c473 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cf122331869145f79fe413b15910c473 2024-11-13T18:12:14,845 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into cf122331869145f79fe413b15910c473(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:14,845 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:14,845 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=13, startTime=1731521534228; duration=0sec 2024-11-13T18:12:14,845 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:14,846 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:14,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521594875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521594875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521594876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:14,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:14,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:14,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/c702b6df845548609f63775d4566fdbf 2024-11-13T18:12:14,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/ead6a663e8114ccd946279cf6f599ef3 is 50, key is test_row_0/B:col10/1731521533341/Put/seqid=0 2024-11-13T18:12:14,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741850_1026 (size=12001) 2024-11-13T18:12:14,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/ead6a663e8114ccd946279cf6f599ef3 2024-11-13T18:12:15,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/934c4a8127bc465baf3b3f7d5887dc82 is 50, key is test_row_0/C:col10/1731521533341/Put/seqid=0 2024-11-13T18:12:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741851_1027 (size=12001) 2024-11-13T18:12:15,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/934c4a8127bc465baf3b3f7d5887dc82 2024-11-13T18:12:15,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/c702b6df845548609f63775d4566fdbf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c702b6df845548609f63775d4566fdbf 2024-11-13T18:12:15,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c702b6df845548609f63775d4566fdbf, entries=150, sequenceid=80, filesize=11.7 K 2024-11-13T18:12:15,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/ead6a663e8114ccd946279cf6f599ef3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ead6a663e8114ccd946279cf6f599ef3 2024-11-13T18:12:15,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ead6a663e8114ccd946279cf6f599ef3, entries=150, sequenceid=80, filesize=11.7 K 2024-11-13T18:12:15,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/934c4a8127bc465baf3b3f7d5887dc82 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/934c4a8127bc465baf3b3f7d5887dc82 2024-11-13T18:12:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:12:15,095 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:15,095 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-13T18:12:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:12:15,099 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:15,100 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:15,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:15,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/934c4a8127bc465baf3b3f7d5887dc82, entries=150, sequenceid=80, filesize=11.7 K 2024-11-13T18:12:15,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 401e2f643e0c1bcf64dcec1f711ca8ed in 608ms, sequenceid=80, compaction requested=false 2024-11-13T18:12:15,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:15,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:12:15,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:15,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:15,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:15,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:15,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:15,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:15,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/b0b49b243de14fbb9c19ef47f3338bc4 is 50, key is test_row_0/A:col10/1731521535186/Put/seqid=0 2024-11-13T18:12:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:12:15,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741852_1028 (size=14341) 2024-11-13T18:12:15,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/b0b49b243de14fbb9c19ef47f3338bc4 2024-11-13T18:12:15,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/d01d5a9464724516bc39dd8e19274935 is 50, key is test_row_0/B:col10/1731521535186/Put/seqid=0 2024-11-13T18:12:15,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521595246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741853_1029 (size=12001) 2024-11-13T18:12:15,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/d01d5a9464724516bc39dd8e19274935 2024-11-13T18:12:15,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521595252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521595252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T18:12:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/3f887aa7095748be8e346c3771d9e2e9 is 50, key is test_row_0/C:col10/1731521535186/Put/seqid=0 2024-11-13T18:12:15,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741854_1030 (size=12001) 2024-11-13T18:12:15,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521595364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521595375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521595375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:12:15,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T18:12:15,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:15,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T18:12:15,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:15,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521595575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521595585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521595586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/3f887aa7095748be8e346c3771d9e2e9 2024-11-13T18:12:15,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/b0b49b243de14fbb9c19ef47f3338bc4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b0b49b243de14fbb9c19ef47f3338bc4 2024-11-13T18:12:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:12:15,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T18:12:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b0b49b243de14fbb9c19ef47f3338bc4, entries=200, sequenceid=97, filesize=14.0 K 2024-11-13T18:12:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:15,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/d01d5a9464724516bc39dd8e19274935 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/d01d5a9464724516bc39dd8e19274935 2024-11-13T18:12:15,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/d01d5a9464724516bc39dd8e19274935, entries=150, sequenceid=97, filesize=11.7 K 2024-11-13T18:12:15,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/3f887aa7095748be8e346c3771d9e2e9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3f887aa7095748be8e346c3771d9e2e9 2024-11-13T18:12:15,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3f887aa7095748be8e346c3771d9e2e9, entries=150, sequenceid=97, filesize=11.7 K 2024-11-13T18:12:15,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 401e2f643e0c1bcf64dcec1f711ca8ed in 614ms, sequenceid=97, compaction requested=true 2024-11-13T18:12:15,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:15,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:15,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:15,806 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:15,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:15,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:15,806 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:15,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:15,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:15,808 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:15,809 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:15,809 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,809 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/bf1f660d96f949e3b29504c58bd374b1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c702b6df845548609f63775d4566fdbf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b0b49b243de14fbb9c19ef47f3338bc4] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=37.5 K 2024-11-13T18:12:15,810 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:15,810 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf1f660d96f949e3b29504c58bd374b1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521532434 2024-11-13T18:12:15,810 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:15,810 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,810 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6f950db1699a4f589f5615b4a2e988df, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ead6a663e8114ccd946279cf6f599ef3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/d01d5a9464724516bc39dd8e19274935] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=35.3 K 2024-11-13T18:12:15,811 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f950db1699a4f589f5615b4a2e988df, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521532434 2024-11-13T18:12:15,811 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c702b6df845548609f63775d4566fdbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731521533336 2024-11-13T18:12:15,818 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ead6a663e8114ccd946279cf6f599ef3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731521533336 2024-11-13T18:12:15,818 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0b49b243de14fbb9c19ef47f3338bc4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731521534524 2024-11-13T18:12:15,818 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d01d5a9464724516bc39dd8e19274935, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731521534524 2024-11-13T18:12:15,854 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:15,855 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/e4d208e6af0e4db6b839f7f087df1a78 is 50, key is test_row_0/B:col10/1731521535186/Put/seqid=0 2024-11-13T18:12:15,857 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#18 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:15,858 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/e22c761ca82f4b7da229b82b9b4619e1 is 50, key is test_row_0/A:col10/1731521535186/Put/seqid=0 2024-11-13T18:12:15,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T18:12:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741855_1031 (size=12207) 2024-11-13T18:12:15,897 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:12:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:15,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f1ab116c889d484a8e5a120e1ce11109 is 50, key is test_row_0/A:col10/1731521535239/Put/seqid=0 2024-11-13T18:12:15,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:15,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741856_1032 (size=12207) 2024-11-13T18:12:15,938 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/e4d208e6af0e4db6b839f7f087df1a78 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e4d208e6af0e4db6b839f7f087df1a78 2024-11-13T18:12:15,939 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/e22c761ca82f4b7da229b82b9b4619e1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/e22c761ca82f4b7da229b82b9b4619e1 2024-11-13T18:12:15,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741857_1033 (size=12001) 2024-11-13T18:12:15,959 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f1ab116c889d484a8e5a120e1ce11109 2024-11-13T18:12:15,964 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into e4d208e6af0e4db6b839f7f087df1a78(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:15,964 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:15,965 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=13, startTime=1731521535806; duration=0sec 2024-11-13T18:12:15,965 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into e22c761ca82f4b7da229b82b9b4619e1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:15,965 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:15,965 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=13, startTime=1731521535806; duration=0sec 2024-11-13T18:12:15,965 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:15,965 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:15,965 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:15,965 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:15,965 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:15,972 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:15,972 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:15,972 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:15,973 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cf122331869145f79fe413b15910c473, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/934c4a8127bc465baf3b3f7d5887dc82, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3f887aa7095748be8e346c3771d9e2e9] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=35.3 K 2024-11-13T18:12:15,974 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting cf122331869145f79fe413b15910c473, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521532434 2024-11-13T18:12:15,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 934c4a8127bc465baf3b3f7d5887dc82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731521533336 2024-11-13T18:12:15,976 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f887aa7095748be8e346c3771d9e2e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731521534524 2024-11-13T18:12:15,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/a0dc95ec9d6f4f53b0d5061c0d8c74e2 is 50, key is test_row_0/B:col10/1731521535239/Put/seqid=0 2024-11-13T18:12:15,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521595973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521595977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521595981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:15,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:15,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,012 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:16,013 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/85a8a06540e94559b23f4f44c82af65b is 50, key is test_row_0/C:col10/1731521535186/Put/seqid=0 2024-11-13T18:12:16,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741858_1034 (size=12001) 2024-11-13T18:12:16,042 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/a0dc95ec9d6f4f53b0d5061c0d8c74e2 2024-11-13T18:12:16,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741859_1035 (size=12207) 2024-11-13T18:12:16,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521596095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,100 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/85a8a06540e94559b23f4f44c82af65b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/85a8a06540e94559b23f4f44c82af65b 2024-11-13T18:12:16,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521596095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/4963938e6cc1452e90993cba3f57f2b5 is 50, key is test_row_0/C:col10/1731521535239/Put/seqid=0 2024-11-13T18:12:16,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521596106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,122 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into 85a8a06540e94559b23f4f44c82af65b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:16,122 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:16,122 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=13, startTime=1731521535806; duration=0sec 2024-11-13T18:12:16,123 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:16,123 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:16,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741860_1036 (size=12001) 2024-11-13T18:12:16,128 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/4963938e6cc1452e90993cba3f57f2b5 2024-11-13T18:12:16,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f1ab116c889d484a8e5a120e1ce11109 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f1ab116c889d484a8e5a120e1ce11109 2024-11-13T18:12:16,153 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f1ab116c889d484a8e5a120e1ce11109, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T18:12:16,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/a0dc95ec9d6f4f53b0d5061c0d8c74e2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/a0dc95ec9d6f4f53b0d5061c0d8c74e2 2024-11-13T18:12:16,172 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/a0dc95ec9d6f4f53b0d5061c0d8c74e2, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T18:12:16,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/4963938e6cc1452e90993cba3f57f2b5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/4963938e6cc1452e90993cba3f57f2b5 2024-11-13T18:12:16,192 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/4963938e6cc1452e90993cba3f57f2b5, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T18:12:16,194 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 401e2f643e0c1bcf64dcec1f711ca8ed in 297ms, sequenceid=113, compaction requested=false 2024-11-13T18:12:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:16,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T18:12:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T18:12:16,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T18:12:16,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0980 sec 2024-11-13T18:12:16,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1110 sec 2024-11-13T18:12:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:12:16,236 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:16,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-13T18:12:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:12:16,248 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:16,251 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:16,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:16,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:12:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:16,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/56715e6f6879474cae96c159e5f98f57 is 50, key is test_row_0/A:col10/1731521535939/Put/seqid=0 2024-11-13T18:12:16,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741861_1037 (size=14491) 2024-11-13T18:12:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:12:16,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521596368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521596373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521596373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:16,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,487 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T18:12:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,488 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T18:12:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521596485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521596485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,490 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:12:16,491 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T18:12:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T18:12:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521596485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,491 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T18:12:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,494 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:12:16,494 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T18:12:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:12:16,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:16,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521596694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521596705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521596705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521596714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,719 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_1' in 401e2f643e0c1bcf64dcec1f711ca8ed of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521596745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:16,746 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_2' in 401e2f643e0c1bcf64dcec1f711ca8ed of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4302 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:16,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/56715e6f6879474cae96c159e5f98f57 2024-11-13T18:12:16,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/18a4441a07dc43b797bdf403a4a93316 is 50, key is test_row_0/B:col10/1731521535939/Put/seqid=0 2024-11-13T18:12:16,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741862_1038 (size=12101) 2024-11-13T18:12:16,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/18a4441a07dc43b797bdf403a4a93316 2024-11-13T18:12:16,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/7343d392c9cf400fa5824c637b3d35ab is 50, key is test_row_0/C:col10/1731521535939/Put/seqid=0 2024-11-13T18:12:16,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741863_1039 (size=12101) 2024-11-13T18:12:16,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/7343d392c9cf400fa5824c637b3d35ab 2024-11-13T18:12:16,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/56715e6f6879474cae96c159e5f98f57 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/56715e6f6879474cae96c159e5f98f57 2024-11-13T18:12:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:12:16,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/56715e6f6879474cae96c159e5f98f57, entries=200, sequenceid=133, filesize=14.2 K 2024-11-13T18:12:16,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/18a4441a07dc43b797bdf403a4a93316 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/18a4441a07dc43b797bdf403a4a93316 2024-11-13T18:12:16,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/18a4441a07dc43b797bdf403a4a93316, entries=150, sequenceid=133, filesize=11.8 K 2024-11-13T18:12:16,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/7343d392c9cf400fa5824c637b3d35ab as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7343d392c9cf400fa5824c637b3d35ab 2024-11-13T18:12:16,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:16,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:16,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7343d392c9cf400fa5824c637b3d35ab, entries=150, sequenceid=133, filesize=11.8 K 2024-11-13T18:12:16,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 401e2f643e0c1bcf64dcec1f711ca8ed in 581ms, sequenceid=133, compaction requested=true 2024-11-13T18:12:16,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:16,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:16,899 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:16,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:16,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:16,899 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:16,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:16,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:16,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:16,901 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:16,901 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:16,901 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:16,901 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,901 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:16,901 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,901 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/e22c761ca82f4b7da229b82b9b4619e1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f1ab116c889d484a8e5a120e1ce11109, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/56715e6f6879474cae96c159e5f98f57] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=37.8 K 2024-11-13T18:12:16,901 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e4d208e6af0e4db6b839f7f087df1a78, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/a0dc95ec9d6f4f53b0d5061c0d8c74e2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/18a4441a07dc43b797bdf403a4a93316] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=35.5 K 2024-11-13T18:12:16,903 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e22c761ca82f4b7da229b82b9b4619e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731521534524 2024-11-13T18:12:16,903 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e4d208e6af0e4db6b839f7f087df1a78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731521534524 2024-11-13T18:12:16,903 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1ab116c889d484a8e5a120e1ce11109, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731521535239 2024-11-13T18:12:16,903 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting a0dc95ec9d6f4f53b0d5061c0d8c74e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731521535239 2024-11-13T18:12:16,904 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56715e6f6879474cae96c159e5f98f57, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731521535939 2024-11-13T18:12:16,904 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a4441a07dc43b797bdf403a4a93316, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731521535939 2024-11-13T18:12:16,922 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:16,923 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/c0438db69b564224812bcd7a27815626 is 50, key is test_row_0/A:col10/1731521535939/Put/seqid=0 2024-11-13T18:12:16,925 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#28 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:16,926 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/9475d10e6fb546bfa684621887056504 is 50, key is test_row_0/B:col10/1731521535939/Put/seqid=0 2024-11-13T18:12:16,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741864_1040 (size=12409) 2024-11-13T18:12:16,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741865_1041 (size=12409) 2024-11-13T18:12:16,963 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/9475d10e6fb546bfa684621887056504 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/9475d10e6fb546bfa684621887056504 2024-11-13T18:12:16,973 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into 9475d10e6fb546bfa684621887056504(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:16,973 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:16,973 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=13, startTime=1731521536899; duration=0sec 2024-11-13T18:12:16,974 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:16,974 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:16,974 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:16,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:16,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:16,976 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:16,976 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/85a8a06540e94559b23f4f44c82af65b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/4963938e6cc1452e90993cba3f57f2b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7343d392c9cf400fa5824c637b3d35ab] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=35.5 K 2024-11-13T18:12:16,976 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a8a06540e94559b23f4f44c82af65b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731521534524 2024-11-13T18:12:16,977 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 4963938e6cc1452e90993cba3f57f2b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731521535239 2024-11-13T18:12:16,978 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 7343d392c9cf400fa5824c637b3d35ab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731521535939 2024-11-13T18:12:16,992 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#29 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:16,993 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/6bdcb84f1f3445849aa8e4f66cc0ad4d is 50, key is test_row_0/C:col10/1731521535939/Put/seqid=0 2024-11-13T18:12:16,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741866_1042 (size=12409) 2024-11-13T18:12:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:17,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:12:17,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:17,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:17,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:17,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:17,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:17,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:17,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/10fc49c45083426db1c096b7e23778f2 is 50, key is test_row_0/A:col10/1731521536339/Put/seqid=0 2024-11-13T18:12:17,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741867_1043 (size=14541) 2024-11-13T18:12:17,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:17,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521597049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521597055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521597057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521597164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521597164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521597165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:17,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:17,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,348 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/c0438db69b564224812bcd7a27815626 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c0438db69b564224812bcd7a27815626 2024-11-13T18:12:17,361 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into c0438db69b564224812bcd7a27815626(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:17,361 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:17,362 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=13, startTime=1731521536899; duration=0sec 2024-11-13T18:12:17,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:17,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:17,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521597374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521597374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521597375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:12:17,411 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/6bdcb84f1f3445849aa8e4f66cc0ad4d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/6bdcb84f1f3445849aa8e4f66cc0ad4d 2024-11-13T18:12:17,421 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into 6bdcb84f1f3445849aa8e4f66cc0ad4d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:17,421 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:17,421 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=13, startTime=1731521536899; duration=0sec 2024-11-13T18:12:17,422 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:17,422 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:17,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/10fc49c45083426db1c096b7e23778f2 2024-11-13T18:12:17,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/492be3abc0bc4d15ac876db9f053c69a is 50, key is test_row_0/B:col10/1731521536339/Put/seqid=0 2024-11-13T18:12:17,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741868_1044 (size=12151) 2024-11-13T18:12:17,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:17,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:17,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:17,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:17,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521597684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521597685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521597685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:17,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:17,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/492be3abc0bc4d15ac876db9f053c69a 2024-11-13T18:12:17,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/337d41868af4474c9d93c21a9736d94a is 50, key is test_row_0/C:col10/1731521536339/Put/seqid=0 2024-11-13T18:12:17,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741869_1045 (size=12151) 2024-11-13T18:12:17,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:17,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:17,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:17,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:18,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:18,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:18,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:18,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:18,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:18,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521598194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:18,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521598194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:18,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:18,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521598195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:18,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:18,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:18,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:18,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/337d41868af4474c9d93c21a9736d94a 2024-11-13T18:12:18,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:18,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:18,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:18,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:18,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:18,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:18,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/10fc49c45083426db1c096b7e23778f2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/10fc49c45083426db1c096b7e23778f2 2024-11-13T18:12:18,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/10fc49c45083426db1c096b7e23778f2, entries=200, sequenceid=151, filesize=14.2 K 2024-11-13T18:12:18,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/492be3abc0bc4d15ac876db9f053c69a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/492be3abc0bc4d15ac876db9f053c69a 2024-11-13T18:12:18,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/492be3abc0bc4d15ac876db9f053c69a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T18:12:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/337d41868af4474c9d93c21a9736d94a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/337d41868af4474c9d93c21a9736d94a 2024-11-13T18:12:18,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/337d41868af4474c9d93c21a9736d94a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T18:12:18,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 401e2f643e0c1bcf64dcec1f711ca8ed in 1325ms, sequenceid=151, compaction requested=false 2024-11-13T18:12:18,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:12:18,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:12:18,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:18,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:12:18,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:18,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:18,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:18,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:18,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:18,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:18,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/5a9a300d168c4d9ba20750e11f59e9c6 is 50, key is test_row_0/A:col10/1731521537046/Put/seqid=0 2024-11-13T18:12:18,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741870_1046 (size=12151) 2024-11-13T18:12:18,872 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/5a9a300d168c4d9ba20750e11f59e9c6 2024-11-13T18:12:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/e6086a2b39c042ef906395059cbbbc24 is 50, key is test_row_0/B:col10/1731521537046/Put/seqid=0 2024-11-13T18:12:18,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741871_1047 (size=12151) 2024-11-13T18:12:18,891 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/e6086a2b39c042ef906395059cbbbc24 2024-11-13T18:12:18,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/acd6219a60d84949ab5d3bf6e1b892e2 is 50, key is test_row_0/C:col10/1731521537046/Put/seqid=0 2024-11-13T18:12:18,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741872_1048 (size=12151) 2024-11-13T18:12:18,911 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/acd6219a60d84949ab5d3bf6e1b892e2 2024-11-13T18:12:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/5a9a300d168c4d9ba20750e11f59e9c6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/5a9a300d168c4d9ba20750e11f59e9c6 2024-11-13T18:12:18,925 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/5a9a300d168c4d9ba20750e11f59e9c6, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T18:12:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/e6086a2b39c042ef906395059cbbbc24 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e6086a2b39c042ef906395059cbbbc24 2024-11-13T18:12:18,940 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e6086a2b39c042ef906395059cbbbc24, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T18:12:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/acd6219a60d84949ab5d3bf6e1b892e2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/acd6219a60d84949ab5d3bf6e1b892e2 2024-11-13T18:12:18,951 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/acd6219a60d84949ab5d3bf6e1b892e2, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T18:12:18,954 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 401e2f643e0c1bcf64dcec1f711ca8ed in 498ms, sequenceid=168, compaction requested=true 2024-11-13T18:12:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T18:12:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T18:12:18,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T18:12:18,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7060 sec 2024-11-13T18:12:18,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.7260 sec 2024-11-13T18:12:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:19,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:12:19,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:19,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:19,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:19,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:19,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:19,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:19,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/6536d7ad3e2e48ef9e05fdcf6a42829e is 50, key is test_row_0/A:col10/1731521539228/Put/seqid=0 2024-11-13T18:12:19,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741873_1049 (size=14541) 2024-11-13T18:12:19,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521599278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:19,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521539256, max timestamp=1731521539285Num uniques 859; 2024-11-13T18:12:19,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:12:19,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521539256, max timestamp=1731521539285Num uniques 859; 2024-11-13T18:12:19,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:12:19,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521539256, max timestamp=1731521539285Num uniques 859; 2024-11-13T18:12:19,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:12:19,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521599380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521599380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:19,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521599394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521599494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521599495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:19,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521599604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:19,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:19,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/6536d7ad3e2e48ef9e05fdcf6a42829e 2024-11-13T18:12:19,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/83e9752c9082467abe23b2ad4b300a43 is 50, key is test_row_0/B:col10/1731521539228/Put/seqid=0 2024-11-13T18:12:19,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741874_1050 (size=12151) 2024-11-13T18:12:19,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/83e9752c9082467abe23b2ad4b300a43 2024-11-13T18:12:19,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/341a59ec48144fd696c5c7d5c73086d8 is 50, key is test_row_0/C:col10/1731521539228/Put/seqid=0 2024-11-13T18:12:19,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741875_1051 (size=12151) 2024-11-13T18:12:19,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/341a59ec48144fd696c5c7d5c73086d8 2024-11-13T18:12:19,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/6536d7ad3e2e48ef9e05fdcf6a42829e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/6536d7ad3e2e48ef9e05fdcf6a42829e 2024-11-13T18:12:19,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/6536d7ad3e2e48ef9e05fdcf6a42829e, entries=200, sequenceid=179, filesize=14.2 K 2024-11-13T18:12:19,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/83e9752c9082467abe23b2ad4b300a43 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/83e9752c9082467abe23b2ad4b300a43 2024-11-13T18:12:19,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/83e9752c9082467abe23b2ad4b300a43, entries=150, sequenceid=179, filesize=11.9 K 2024-11-13T18:12:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521599705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521599706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:19,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/341a59ec48144fd696c5c7d5c73086d8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/341a59ec48144fd696c5c7d5c73086d8 2024-11-13T18:12:19,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/341a59ec48144fd696c5c7d5c73086d8, entries=150, sequenceid=179, filesize=11.9 K 2024-11-13T18:12:19,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 401e2f643e0c1bcf64dcec1f711ca8ed in 490ms, sequenceid=179, compaction requested=true 2024-11-13T18:12:19,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:19,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:19,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:19,720 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:19,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:19,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:19,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:19,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:19,721 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:19,722 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53642 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:19,722 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:19,722 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:19,722 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c0438db69b564224812bcd7a27815626, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/10fc49c45083426db1c096b7e23778f2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/5a9a300d168c4d9ba20750e11f59e9c6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/6536d7ad3e2e48ef9e05fdcf6a42829e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=52.4 K 2024-11-13T18:12:19,723 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0438db69b564224812bcd7a27815626, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731521535939 2024-11-13T18:12:19,723 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:19,723 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:19,723 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:19,723 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10fc49c45083426db1c096b7e23778f2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731521536339 2024-11-13T18:12:19,723 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/9475d10e6fb546bfa684621887056504, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/492be3abc0bc4d15ac876db9f053c69a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e6086a2b39c042ef906395059cbbbc24, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/83e9752c9082467abe23b2ad4b300a43] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=47.7 K 2024-11-13T18:12:19,724 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a9a300d168c4d9ba20750e11f59e9c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731521537046 2024-11-13T18:12:19,724 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 9475d10e6fb546bfa684621887056504, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731521535939 2024-11-13T18:12:19,724 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6536d7ad3e2e48ef9e05fdcf6a42829e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731521539215 2024-11-13T18:12:19,724 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 492be3abc0bc4d15ac876db9f053c69a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731521536339 2024-11-13T18:12:19,725 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e6086a2b39c042ef906395059cbbbc24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731521537046 2024-11-13T18:12:19,726 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e9752c9082467abe23b2ad4b300a43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731521539215 2024-11-13T18:12:19,741 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:19,742 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:19,742 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/ec298fc400a44daa9e2ab0435268c07c is 50, key is test_row_0/B:col10/1731521539228/Put/seqid=0 2024-11-13T18:12:19,742 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/e76b97b27cc740ad9867c4f1ea92c238 is 50, key is test_row_0/A:col10/1731521539228/Put/seqid=0 2024-11-13T18:12:19,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741876_1052 (size=12595) 2024-11-13T18:12:19,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741877_1053 (size=12595) 2024-11-13T18:12:19,766 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/e76b97b27cc740ad9867c4f1ea92c238 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/e76b97b27cc740ad9867c4f1ea92c238 2024-11-13T18:12:19,774 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into e76b97b27cc740ad9867c4f1ea92c238(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:19,774 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:19,774 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=12, startTime=1731521539719; duration=0sec 2024-11-13T18:12:19,775 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:19,775 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:19,775 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:19,778 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:19,778 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:19,778 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:19,778 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/6bdcb84f1f3445849aa8e4f66cc0ad4d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/337d41868af4474c9d93c21a9736d94a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/acd6219a60d84949ab5d3bf6e1b892e2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/341a59ec48144fd696c5c7d5c73086d8] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=47.7 K 2024-11-13T18:12:19,779 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bdcb84f1f3445849aa8e4f66cc0ad4d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731521535939 2024-11-13T18:12:19,779 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 337d41868af4474c9d93c21a9736d94a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731521536339 2024-11-13T18:12:19,780 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting acd6219a60d84949ab5d3bf6e1b892e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731521537046 2024-11-13T18:12:19,780 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 341a59ec48144fd696c5c7d5c73086d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731521539215 2024-11-13T18:12:19,792 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:19,793 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/30b10a3cf00b410db7072952ac98f8fe is 50, key is test_row_0/C:col10/1731521539228/Put/seqid=0 2024-11-13T18:12:19,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741878_1054 (size=12595) 2024-11-13T18:12:19,831 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/30b10a3cf00b410db7072952ac98f8fe as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/30b10a3cf00b410db7072952ac98f8fe 2024-11-13T18:12:19,839 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into 30b10a3cf00b410db7072952ac98f8fe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:19,839 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:19,839 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=12, startTime=1731521539720; duration=0sec 2024-11-13T18:12:19,839 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:19,839 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:19,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T18:12:19,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:19,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:19,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:19,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:19,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:19,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:19,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/aa1065fe1cc94cfeb880b6c4fc38ad52 is 50, key is test_row_0/A:col10/1731521539927/Put/seqid=0 2024-11-13T18:12:19,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741879_1055 (size=14541) 2024-11-13T18:12:19,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521599983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521600014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521600014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521600094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,162 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/ec298fc400a44daa9e2ab0435268c07c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ec298fc400a44daa9e2ab0435268c07c 2024-11-13T18:12:20,172 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into ec298fc400a44daa9e2ab0435268c07c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:20,172 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:20,172 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=12, startTime=1731521539720; duration=0sec 2024-11-13T18:12:20,172 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:20,172 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:20,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521600305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/aa1065fe1cc94cfeb880b6c4fc38ad52 2024-11-13T18:12:20,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/fabff8fba00844499680550e2b03b86a is 50, key is test_row_0/B:col10/1731521539927/Put/seqid=0 2024-11-13T18:12:20,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741880_1056 (size=12151) 2024-11-13T18:12:20,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/fabff8fba00844499680550e2b03b86a 2024-11-13T18:12:20,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/5bf1f707ea304d6db72745af4deff7cd is 50, key is test_row_0/C:col10/1731521539927/Put/seqid=0 2024-11-13T18:12:20,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741881_1057 (size=12151) 2024-11-13T18:12:20,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/5bf1f707ea304d6db72745af4deff7cd 2024-11-13T18:12:20,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/aa1065fe1cc94cfeb880b6c4fc38ad52 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/aa1065fe1cc94cfeb880b6c4fc38ad52 2024-11-13T18:12:20,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/aa1065fe1cc94cfeb880b6c4fc38ad52, entries=200, sequenceid=217, filesize=14.2 K 2024-11-13T18:12:20,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/fabff8fba00844499680550e2b03b86a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/fabff8fba00844499680550e2b03b86a 2024-11-13T18:12:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:12:20,414 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:20,415 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:20,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/fabff8fba00844499680550e2b03b86a, entries=150, sequenceid=217, filesize=11.9 K 2024-11-13T18:12:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-13T18:12:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:12:20,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/5bf1f707ea304d6db72745af4deff7cd as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5bf1f707ea304d6db72745af4deff7cd 2024-11-13T18:12:20,420 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:20,421 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:20,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:20,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5bf1f707ea304d6db72745af4deff7cd, entries=150, sequenceid=217, filesize=11.9 K 2024-11-13T18:12:20,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 401e2f643e0c1bcf64dcec1f711ca8ed in 500ms, sequenceid=217, compaction requested=false 2024-11-13T18:12:20,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:12:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:20,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:12:20,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:20,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:20,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:20,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:20,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:20,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:20,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/67548bb64bc048169fed135f956260e8 is 50, key is test_row_0/A:col10/1731521540526/Put/seqid=0 2024-11-13T18:12:20,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741882_1058 (size=12151) 2024-11-13T18:12:20,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:20,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:20,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:20,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:20,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:20,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521600613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521600614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521600614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521600724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521600724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:20,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:20,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:20,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:20,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:20,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:12:20,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521600734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,736 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_1' in 401e2f643e0c1bcf64dcec1f711ca8ed of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521600764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,766 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_2' in 401e2f643e0c1bcf64dcec1f711ca8ed of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8322 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:20,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:20,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:20,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:20,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:20,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:20,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521600935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521600935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:20,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/67548bb64bc048169fed135f956260e8 2024-11-13T18:12:20,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/f4357dd7388540ea9d07cf83702c1389 is 50, key is test_row_0/B:col10/1731521540526/Put/seqid=0 2024-11-13T18:12:20,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741883_1059 (size=12151) 2024-11-13T18:12:20,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/f4357dd7388540ea9d07cf83702c1389 2024-11-13T18:12:20,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/cb84c32c65304abaabe2ad026f1e41ef is 50, key is test_row_0/C:col10/1731521540526/Put/seqid=0 2024-11-13T18:12:20,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741884_1060 (size=12151) 2024-11-13T18:12:20,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/cb84c32c65304abaabe2ad026f1e41ef 2024-11-13T18:12:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/67548bb64bc048169fed135f956260e8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/67548bb64bc048169fed135f956260e8 2024-11-13T18:12:21,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/67548bb64bc048169fed135f956260e8, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T18:12:21,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/f4357dd7388540ea9d07cf83702c1389 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f4357dd7388540ea9d07cf83702c1389 2024-11-13T18:12:21,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f4357dd7388540ea9d07cf83702c1389, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T18:12:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/cb84c32c65304abaabe2ad026f1e41ef as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cb84c32c65304abaabe2ad026f1e41ef 2024-11-13T18:12:21,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cb84c32c65304abaabe2ad026f1e41ef, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T18:12:21,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:21,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:21,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 401e2f643e0c1bcf64dcec1f711ca8ed in 513ms, sequenceid=230, compaction requested=true 2024-11-13T18:12:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:21,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:21,042 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:21,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:21,042 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:12:21,047 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:21,047 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:21,047 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,047 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ec298fc400a44daa9e2ab0435268c07c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/fabff8fba00844499680550e2b03b86a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f4357dd7388540ea9d07cf83702c1389] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=36.0 K 2024-11-13T18:12:21,048 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ec298fc400a44daa9e2ab0435268c07c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731521539215 2024-11-13T18:12:21,048 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:21,049 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:21,049 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,049 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/e76b97b27cc740ad9867c4f1ea92c238, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/aa1065fe1cc94cfeb880b6c4fc38ad52, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/67548bb64bc048169fed135f956260e8] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=38.4 K 2024-11-13T18:12:21,049 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting fabff8fba00844499680550e2b03b86a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731521539335 2024-11-13T18:12:21,050 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e76b97b27cc740ad9867c4f1ea92c238, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731521539215 2024-11-13T18:12:21,050 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f4357dd7388540ea9d07cf83702c1389, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521539972 2024-11-13T18:12:21,050 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa1065fe1cc94cfeb880b6c4fc38ad52, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731521539319 2024-11-13T18:12:21,051 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67548bb64bc048169fed135f956260e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521539972 2024-11-13T18:12:21,064 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:21,065 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/21acf727509440d28da0106125b48c59 is 50, key is test_row_0/B:col10/1731521540526/Put/seqid=0 2024-11-13T18:12:21,072 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#49 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:21,073 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/58c54be8ca864a9b90ca315895ed4f88 is 50, key is test_row_0/A:col10/1731521540526/Put/seqid=0 2024-11-13T18:12:21,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741885_1061 (size=12697) 2024-11-13T18:12:21,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741886_1062 (size=12697) 2024-11-13T18:12:21,093 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/21acf727509440d28da0106125b48c59 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/21acf727509440d28da0106125b48c59 2024-11-13T18:12:21,095 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/58c54be8ca864a9b90ca315895ed4f88 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/58c54be8ca864a9b90ca315895ed4f88 2024-11-13T18:12:21,103 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into 58c54be8ca864a9b90ca315895ed4f88(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:21,103 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into 21acf727509440d28da0106125b48c59(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:21,103 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:21,103 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:21,103 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=13, startTime=1731521541042; duration=0sec 2024-11-13T18:12:21,104 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=13, startTime=1731521541042; duration=0sec 2024-11-13T18:12:21,104 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:21,104 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:21,104 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:21,104 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:21,104 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:21,113 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:21,113 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:21,113 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,113 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/30b10a3cf00b410db7072952ac98f8fe, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5bf1f707ea304d6db72745af4deff7cd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cb84c32c65304abaabe2ad026f1e41ef] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=36.0 K 2024-11-13T18:12:21,114 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30b10a3cf00b410db7072952ac98f8fe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731521539215 2024-11-13T18:12:21,114 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bf1f707ea304d6db72745af4deff7cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731521539335 2024-11-13T18:12:21,115 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb84c32c65304abaabe2ad026f1e41ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521539972 2024-11-13T18:12:21,124 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:21,124 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/661296b2669248178a0793366c1711d4 is 50, key is test_row_0/C:col10/1731521540526/Put/seqid=0 2024-11-13T18:12:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:21,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T18:12:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:21,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521540599, max timestamp=1731521541126Num uniques 859; 2024-11-13T18:12:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:21,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:12:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:21,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:21,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:21,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:21,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741887_1063 (size=12697) 2024-11-13T18:12:21,146 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/661296b2669248178a0793366c1711d4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/661296b2669248178a0793366c1711d4 2024-11-13T18:12:21,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/7fe52c34dad44b879273566215cc83a0 is 50, key is test_row_0/A:col10/1731521540599/Put/seqid=0 2024-11-13T18:12:21,155 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into 661296b2669248178a0793366c1711d4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:21,155 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:21,155 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=13, startTime=1731521541043; duration=0sec 2024-11-13T18:12:21,155 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:21,155 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:21,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741888_1064 (size=12151) 2024-11-13T18:12:21,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/7fe52c34dad44b879273566215cc83a0 2024-11-13T18:12:21,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/44aeefc419f54f5194a40eb615f7fc70 is 50, key is test_row_0/B:col10/1731521540599/Put/seqid=0 2024-11-13T18:12:21,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741889_1065 (size=12151) 2024-11-13T18:12:21,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/44aeefc419f54f5194a40eb615f7fc70 2024-11-13T18:12:21,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/39d8e35b02f94a7eb28601cee8da52f0 is 50, key is test_row_0/C:col10/1731521540599/Put/seqid=0 2024-11-13T18:12:21,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741890_1066 (size=12151) 2024-11-13T18:12:21,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521601220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:21,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521601244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:21,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521601244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:21,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521601334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:21,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:21,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:21,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:21,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:21,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521601545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:12:21,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/39d8e35b02f94a7eb28601cee8da52f0 2024-11-13T18:12:21,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/7fe52c34dad44b879273566215cc83a0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7fe52c34dad44b879273566215cc83a0 2024-11-13T18:12:21,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7fe52c34dad44b879273566215cc83a0, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T18:12:21,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/44aeefc419f54f5194a40eb615f7fc70 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/44aeefc419f54f5194a40eb615f7fc70 2024-11-13T18:12:21,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/44aeefc419f54f5194a40eb615f7fc70, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T18:12:21,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/39d8e35b02f94a7eb28601cee8da52f0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/39d8e35b02f94a7eb28601cee8da52f0 2024-11-13T18:12:21,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/39d8e35b02f94a7eb28601cee8da52f0, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T18:12:21,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:21,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:21,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 401e2f643e0c1bcf64dcec1f711ca8ed in 528ms, sequenceid=253, compaction requested=false 2024-11-13T18:12:21,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:21,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:21,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:12:21,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:21,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:21,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:21,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:21,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:21,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:21,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/b4e5010b5f0e483fabcc186fc473229f is 50, key is test_row_0/A:col10/1731521541187/Put/seqid=0 2024-11-13T18:12:21,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741891_1067 (size=14741) 2024-11-13T18:12:21,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/b4e5010b5f0e483fabcc186fc473229f 2024-11-13T18:12:21,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/46bd0934500a4c4ebd90350c254f93fd is 50, key is test_row_0/B:col10/1731521541187/Put/seqid=0 2024-11-13T18:12:21,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521601799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:21,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521601802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:21,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741892_1068 (size=12301) 2024-11-13T18:12:21,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:21,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/46bd0934500a4c4ebd90350c254f93fd 2024-11-13T18:12:21,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:21,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:21,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/fd88f9db32fe4243af4a2381b87b2330 is 50, key is test_row_0/C:col10/1731521541187/Put/seqid=0 2024-11-13T18:12:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741893_1069 (size=12301) 2024-11-13T18:12:21,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521601855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:21,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521601914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521601915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:21,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:21,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:21,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:21,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:21,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:22,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:22,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:22,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521602124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521602125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/fd88f9db32fe4243af4a2381b87b2330 2024-11-13T18:12:22,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/b4e5010b5f0e483fabcc186fc473229f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b4e5010b5f0e483fabcc186fc473229f 2024-11-13T18:12:22,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b4e5010b5f0e483fabcc186fc473229f, entries=200, sequenceid=271, filesize=14.4 K 2024-11-13T18:12:22,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/46bd0934500a4c4ebd90350c254f93fd as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/46bd0934500a4c4ebd90350c254f93fd 2024-11-13T18:12:22,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/46bd0934500a4c4ebd90350c254f93fd, entries=150, sequenceid=271, filesize=12.0 K 2024-11-13T18:12:22,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/fd88f9db32fe4243af4a2381b87b2330 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd88f9db32fe4243af4a2381b87b2330 2024-11-13T18:12:22,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd88f9db32fe4243af4a2381b87b2330, entries=150, sequenceid=271, filesize=12.0 K 2024-11-13T18:12:22,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 401e2f643e0c1bcf64dcec1f711ca8ed in 501ms, sequenceid=271, compaction requested=true 2024-11-13T18:12:22,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:22,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:22,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:22,270 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:22,270 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:22,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:22,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:22,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:22,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:22,271 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:22,271 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:22,271 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:22,271 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/21acf727509440d28da0106125b48c59, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/44aeefc419f54f5194a40eb615f7fc70, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/46bd0934500a4c4ebd90350c254f93fd] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=36.3 K 2024-11-13T18:12:22,272 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:22,272 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:22,272 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:22,272 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/58c54be8ca864a9b90ca315895ed4f88, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7fe52c34dad44b879273566215cc83a0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b4e5010b5f0e483fabcc186fc473229f] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=38.7 K 2024-11-13T18:12:22,273 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 21acf727509440d28da0106125b48c59, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521539972 2024-11-13T18:12:22,273 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58c54be8ca864a9b90ca315895ed4f88, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521539972 2024-11-13T18:12:22,273 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 44aeefc419f54f5194a40eb615f7fc70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731521540599 2024-11-13T18:12:22,274 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe52c34dad44b879273566215cc83a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731521540599 2024-11-13T18:12:22,274 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 46bd0934500a4c4ebd90350c254f93fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731521541187 2024-11-13T18:12:22,274 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4e5010b5f0e483fabcc186fc473229f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731521541187 2024-11-13T18:12:22,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:12:22,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:22,279 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:12:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/c9d1a36eb59b483589b7704ecfe03091 is 50, key is test_row_0/A:col10/1731521541797/Put/seqid=0 2024-11-13T18:12:22,294 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#58 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:22,294 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/e761641aa9094beeb26ca34d133bacb7 is 50, key is test_row_0/B:col10/1731521541187/Put/seqid=0 2024-11-13T18:12:22,298 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#59 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:22,298 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/29e49d0056bd4df5a6ea5768a4f353e5 is 50, key is test_row_0/A:col10/1731521541187/Put/seqid=0 2024-11-13T18:12:22,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741894_1070 (size=12301) 2024-11-13T18:12:22,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741895_1071 (size=12949) 2024-11-13T18:12:22,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741896_1072 (size=12949) 2024-11-13T18:12:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:22,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:22,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521602467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521602474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521602477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:12:22,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521602594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521602604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521602604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,700 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/c9d1a36eb59b483589b7704ecfe03091 2024-11-13T18:12:22,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/20e37c193b454664bf160f9208f57863 is 50, key is test_row_0/B:col10/1731521541797/Put/seqid=0 2024-11-13T18:12:22,719 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/e761641aa9094beeb26ca34d133bacb7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e761641aa9094beeb26ca34d133bacb7 2024-11-13T18:12:22,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741897_1073 (size=12301) 2024-11-13T18:12:22,726 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/20e37c193b454664bf160f9208f57863 2024-11-13T18:12:22,728 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into e761641aa9094beeb26ca34d133bacb7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:22,728 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:22,728 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=13, startTime=1731521542270; duration=0sec 2024-11-13T18:12:22,729 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:22,729 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:22,729 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:22,735 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:22,735 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:22,735 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:22,736 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/661296b2669248178a0793366c1711d4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/39d8e35b02f94a7eb28601cee8da52f0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd88f9db32fe4243af4a2381b87b2330] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=36.3 K 2024-11-13T18:12:22,738 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/29e49d0056bd4df5a6ea5768a4f353e5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/29e49d0056bd4df5a6ea5768a4f353e5 2024-11-13T18:12:22,738 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 661296b2669248178a0793366c1711d4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521539972 2024-11-13T18:12:22,739 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 39d8e35b02f94a7eb28601cee8da52f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731521540599 2024-11-13T18:12:22,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/ca7b3b930a5d46ea8d853d006e3ebf21 is 50, key is test_row_0/C:col10/1731521541797/Put/seqid=0 2024-11-13T18:12:22,742 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting fd88f9db32fe4243af4a2381b87b2330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731521541187 2024-11-13T18:12:22,749 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into 29e49d0056bd4df5a6ea5768a4f353e5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:22,750 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:22,750 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=13, startTime=1731521542269; duration=0sec 2024-11-13T18:12:22,750 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:22,750 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:22,760 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:22,761 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/5215f66c9301476db14c52c2d69a2604 is 50, key is test_row_0/C:col10/1731521541187/Put/seqid=0 2024-11-13T18:12:22,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741898_1074 (size=12301) 2024-11-13T18:12:22,765 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/ca7b3b930a5d46ea8d853d006e3ebf21 2024-11-13T18:12:22,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741899_1075 (size=12949) 2024-11-13T18:12:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/c9d1a36eb59b483589b7704ecfe03091 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c9d1a36eb59b483589b7704ecfe03091 2024-11-13T18:12:22,781 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c9d1a36eb59b483589b7704ecfe03091, entries=150, sequenceid=286, filesize=12.0 K 2024-11-13T18:12:22,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/20e37c193b454664bf160f9208f57863 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/20e37c193b454664bf160f9208f57863 2024-11-13T18:12:22,788 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/20e37c193b454664bf160f9208f57863, entries=150, sequenceid=286, filesize=12.0 K 2024-11-13T18:12:22,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/ca7b3b930a5d46ea8d853d006e3ebf21 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ca7b3b930a5d46ea8d853d006e3ebf21 2024-11-13T18:12:22,796 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ca7b3b930a5d46ea8d853d006e3ebf21, entries=150, sequenceid=286, filesize=12.0 K 2024-11-13T18:12:22,798 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 401e2f643e0c1bcf64dcec1f711ca8ed in 520ms, sequenceid=286, compaction requested=false 2024-11-13T18:12:22,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:22,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:22,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T18:12:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T18:12:22,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T18:12:22,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3780 sec 2024-11-13T18:12:22,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.3890 sec 2024-11-13T18:12:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:22,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:12:22,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:22,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:22,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:22,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:22,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:22,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:22,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/3cf9620aa7054318bcb0fce928a67c55 is 50, key is test_row_0/A:col10/1731521542456/Put/seqid=0 2024-11-13T18:12:22,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741900_1076 (size=12301) 2024-11-13T18:12:22,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521602838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521602842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521602843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521602954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521602954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521602955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:23,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521603164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521603164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:23,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:23,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521603164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:23,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:23,177 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/5215f66c9301476db14c52c2d69a2604 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5215f66c9301476db14c52c2d69a2604 2024-11-13T18:12:23,186 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into 5215f66c9301476db14c52c2d69a2604(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:23,186 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:23,186 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=13, startTime=1731521542270; duration=0sec 2024-11-13T18:12:23,186 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:23,186 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:23,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/3cf9620aa7054318bcb0fce928a67c55 2024-11-13T18:12:23,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/54b54c3b3d4645d19224ffe367b27a2e is 50, key is test_row_0/B:col10/1731521542456/Put/seqid=0 2024-11-13T18:12:23,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741901_1077 (size=12301) 2024-11-13T18:12:23,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/54b54c3b3d4645d19224ffe367b27a2e 2024-11-13T18:12:23,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/fd6e1ca50b8a4e829bcc09ec88525f60 is 50, key is test_row_0/C:col10/1731521542456/Put/seqid=0 2024-11-13T18:12:23,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741902_1078 (size=12301) 2024-11-13T18:12:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521603474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521603474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521603474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:23,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/fd6e1ca50b8a4e829bcc09ec88525f60 2024-11-13T18:12:23,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/3cf9620aa7054318bcb0fce928a67c55 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/3cf9620aa7054318bcb0fce928a67c55 2024-11-13T18:12:23,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/3cf9620aa7054318bcb0fce928a67c55, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T18:12:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/54b54c3b3d4645d19224ffe367b27a2e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/54b54c3b3d4645d19224ffe367b27a2e 2024-11-13T18:12:23,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/54b54c3b3d4645d19224ffe367b27a2e, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T18:12:23,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/fd6e1ca50b8a4e829bcc09ec88525f60 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd6e1ca50b8a4e829bcc09ec88525f60 2024-11-13T18:12:23,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd6e1ca50b8a4e829bcc09ec88525f60, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T18:12:23,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 401e2f643e0c1bcf64dcec1f711ca8ed in 907ms, sequenceid=306, compaction requested=true 2024-11-13T18:12:23,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:23,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:23,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:23,716 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:23,716 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:23,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:23,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:23,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:23,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:23,717 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:23,717 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:23,717 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:23,718 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:23,718 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:23,718 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:23,718 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/29e49d0056bd4df5a6ea5768a4f353e5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c9d1a36eb59b483589b7704ecfe03091, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/3cf9620aa7054318bcb0fce928a67c55] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=36.7 K 2024-11-13T18:12:23,718 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e761641aa9094beeb26ca34d133bacb7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/20e37c193b454664bf160f9208f57863, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/54b54c3b3d4645d19224ffe367b27a2e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=36.7 K 2024-11-13T18:12:23,718 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29e49d0056bd4df5a6ea5768a4f353e5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731521541187 2024-11-13T18:12:23,718 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e761641aa9094beeb26ca34d133bacb7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731521541187 2024-11-13T18:12:23,719 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9d1a36eb59b483589b7704ecfe03091, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731521541792 2024-11-13T18:12:23,719 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e37c193b454664bf160f9208f57863, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731521541792 2024-11-13T18:12:23,719 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cf9620aa7054318bcb0fce928a67c55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731521542453 2024-11-13T18:12:23,719 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 54b54c3b3d4645d19224ffe367b27a2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731521542453 2024-11-13T18:12:23,729 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:23,730 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/0e5df77be6f245b693274942e28d8ab1 is 50, key is test_row_0/A:col10/1731521542456/Put/seqid=0 2024-11-13T18:12:23,731 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:23,733 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/56c3427bbd61492e97f120de22340664 is 50, key is test_row_0/B:col10/1731521542456/Put/seqid=0 2024-11-13T18:12:23,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741903_1079 (size=13051) 2024-11-13T18:12:23,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741904_1080 (size=13051) 2024-11-13T18:12:23,746 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/0e5df77be6f245b693274942e28d8ab1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/0e5df77be6f245b693274942e28d8ab1 2024-11-13T18:12:23,752 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into 0e5df77be6f245b693274942e28d8ab1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:23,753 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:23,753 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=13, startTime=1731521543716; duration=0sec 2024-11-13T18:12:23,753 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:23,753 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:23,753 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:23,754 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:23,754 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:23,754 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:23,754 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5215f66c9301476db14c52c2d69a2604, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ca7b3b930a5d46ea8d853d006e3ebf21, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd6e1ca50b8a4e829bcc09ec88525f60] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=36.7 K 2024-11-13T18:12:23,755 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5215f66c9301476db14c52c2d69a2604, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731521541187 2024-11-13T18:12:23,756 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca7b3b930a5d46ea8d853d006e3ebf21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731521541792 2024-11-13T18:12:23,756 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd6e1ca50b8a4e829bcc09ec88525f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731521542453 2024-11-13T18:12:23,767 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:23,768 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/f12f675ddd6d4aabbb8dde37d259d342 is 50, key is test_row_0/C:col10/1731521542456/Put/seqid=0 2024-11-13T18:12:23,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741905_1081 (size=13051) 2024-11-13T18:12:23,784 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/f12f675ddd6d4aabbb8dde37d259d342 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/f12f675ddd6d4aabbb8dde37d259d342 2024-11-13T18:12:23,810 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into f12f675ddd6d4aabbb8dde37d259d342(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:23,810 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:23,810 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=13, startTime=1731521543717; duration=0sec 2024-11-13T18:12:23,810 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:23,810 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:23,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:12:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:24,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/553200f7fcb5414cb24b1808f93a3a38 is 50, key is test_row_0/A:col10/1731521542841/Put/seqid=0 2024-11-13T18:12:24,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741906_1082 (size=12301) 2024-11-13T18:12:24,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/553200f7fcb5414cb24b1808f93a3a38 2024-11-13T18:12:24,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/370963797e694597b0fc199b6b3467cc is 50, key is test_row_0/B:col10/1731521542841/Put/seqid=0 2024-11-13T18:12:24,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521604038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521604039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741907_1083 (size=12301) 2024-11-13T18:12:24,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521604040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/370963797e694597b0fc199b6b3467cc 2024-11-13T18:12:24,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/d6718fc5c676428ebac4abff9d7c8084 is 50, key is test_row_0/C:col10/1731521542841/Put/seqid=0 2024-11-13T18:12:24,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741908_1084 (size=12301) 2024-11-13T18:12:24,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/d6718fc5c676428ebac4abff9d7c8084 2024-11-13T18:12:24,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/553200f7fcb5414cb24b1808f93a3a38 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/553200f7fcb5414cb24b1808f93a3a38 2024-11-13T18:12:24,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/553200f7fcb5414cb24b1808f93a3a38, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T18:12:24,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/370963797e694597b0fc199b6b3467cc as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/370963797e694597b0fc199b6b3467cc 2024-11-13T18:12:24,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/370963797e694597b0fc199b6b3467cc, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T18:12:24,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/d6718fc5c676428ebac4abff9d7c8084 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/d6718fc5c676428ebac4abff9d7c8084 2024-11-13T18:12:24,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/d6718fc5c676428ebac4abff9d7c8084, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T18:12:24,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 401e2f643e0c1bcf64dcec1f711ca8ed in 101ms, sequenceid=326, compaction requested=false 2024-11-13T18:12:24,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:24,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:12:24,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:24,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:24,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:24,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:24,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:24,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:24,151 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/56c3427bbd61492e97f120de22340664 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/56c3427bbd61492e97f120de22340664 2024-11-13T18:12:24,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/57e8746ddb0b4b4db2d239aa2a5515d1 is 50, key is test_row_0/A:col10/1731521544144/Put/seqid=0 2024-11-13T18:12:24,161 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into 56c3427bbd61492e97f120de22340664(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:24,161 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:24,161 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=13, startTime=1731521543716; duration=0sec 2024-11-13T18:12:24,162 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:24,162 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:24,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741909_1085 (size=14741) 2024-11-13T18:12:24,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/57e8746ddb0b4b4db2d239aa2a5515d1 2024-11-13T18:12:24,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/85bcb2d508ee44258072a37dbcd907a7 is 50, key is test_row_0/B:col10/1731521544144/Put/seqid=0 2024-11-13T18:12:24,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521604170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741910_1086 (size=12301) 2024-11-13T18:12:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/85bcb2d508ee44258072a37dbcd907a7 2024-11-13T18:12:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521604176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521604178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/fdbeebf9affa4d2daedf97ab6a6a9b8e is 50, key is test_row_0/C:col10/1731521544144/Put/seqid=0 2024-11-13T18:12:24,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741911_1087 (size=12301) 2024-11-13T18:12:24,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/fdbeebf9affa4d2daedf97ab6a6a9b8e 2024-11-13T18:12:24,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/57e8746ddb0b4b4db2d239aa2a5515d1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/57e8746ddb0b4b4db2d239aa2a5515d1 2024-11-13T18:12:24,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/57e8746ddb0b4b4db2d239aa2a5515d1, entries=200, sequenceid=343, filesize=14.4 K 2024-11-13T18:12:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/85bcb2d508ee44258072a37dbcd907a7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/85bcb2d508ee44258072a37dbcd907a7 2024-11-13T18:12:24,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/85bcb2d508ee44258072a37dbcd907a7, entries=150, sequenceid=343, filesize=12.0 K 2024-11-13T18:12:24,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/fdbeebf9affa4d2daedf97ab6a6a9b8e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fdbeebf9affa4d2daedf97ab6a6a9b8e 2024-11-13T18:12:24,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fdbeebf9affa4d2daedf97ab6a6a9b8e, entries=150, sequenceid=343, filesize=12.0 K 2024-11-13T18:12:24,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 401e2f643e0c1bcf64dcec1f711ca8ed in 81ms, sequenceid=343, compaction requested=true 2024-11-13T18:12:24,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:24,228 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:24,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:24,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:24,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:24,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:24,229 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:24,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:24,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:24,230 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:24,230 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:24,230 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:24,230 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/0e5df77be6f245b693274942e28d8ab1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/553200f7fcb5414cb24b1808f93a3a38, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/57e8746ddb0b4b4db2d239aa2a5515d1] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=39.2 K 2024-11-13T18:12:24,231 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:24,231 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e5df77be6f245b693274942e28d8ab1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731521542453 2024-11-13T18:12:24,231 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:24,231 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:24,231 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/56c3427bbd61492e97f120de22340664, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/370963797e694597b0fc199b6b3467cc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/85bcb2d508ee44258072a37dbcd907a7] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=36.8 K 2024-11-13T18:12:24,232 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 553200f7fcb5414cb24b1808f93a3a38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731521542841 2024-11-13T18:12:24,232 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 56c3427bbd61492e97f120de22340664, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731521542453 2024-11-13T18:12:24,232 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e8746ddb0b4b4db2d239aa2a5515d1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731521544023 2024-11-13T18:12:24,233 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 370963797e694597b0fc199b6b3467cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731521542841 2024-11-13T18:12:24,234 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 85bcb2d508ee44258072a37dbcd907a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731521544027 2024-11-13T18:12:24,244 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:24,245 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/7748e071f0794dd2907c785e5997263a is 50, key is test_row_0/A:col10/1731521544144/Put/seqid=0 2024-11-13T18:12:24,249 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:24,250 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/6699919de02945bea18133b61c18fa4d is 50, key is test_row_0/B:col10/1731521544144/Put/seqid=0 2024-11-13T18:12:24,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741912_1088 (size=13153) 2024-11-13T18:12:24,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741913_1089 (size=13153) 2024-11-13T18:12:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:24,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:12:24,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:24,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:24,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:24,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:24,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:24,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:24,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/27b77c5cd61441069c77e511a07b92bd is 50, key is test_row_0/A:col10/1731521544149/Put/seqid=0 2024-11-13T18:12:24,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741914_1090 (size=14741) 2024-11-13T18:12:24,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521604314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521604314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521604316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521604424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521604424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521604424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:12:24,574 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:24,575 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-13T18:12:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:12:24,578 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:24,579 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:24,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:24,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521604634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521604635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521604635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,667 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/7748e071f0794dd2907c785e5997263a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7748e071f0794dd2907c785e5997263a 2024-11-13T18:12:24,669 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/6699919de02945bea18133b61c18fa4d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6699919de02945bea18133b61c18fa4d 2024-11-13T18:12:24,679 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into 6699919de02945bea18133b61c18fa4d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:24,679 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:24,679 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=13, startTime=1731521544229; duration=0sec 2024-11-13T18:12:24,679 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:24,679 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:24,680 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:24,680 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into 7748e071f0794dd2907c785e5997263a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:24,681 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:24,681 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=13, startTime=1731521544228; duration=0sec 2024-11-13T18:12:24,681 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:24,681 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:24,681 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:24,681 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:24,681 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:24,682 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/f12f675ddd6d4aabbb8dde37d259d342, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/d6718fc5c676428ebac4abff9d7c8084, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fdbeebf9affa4d2daedf97ab6a6a9b8e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=36.8 K 2024-11-13T18:12:24,682 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f12f675ddd6d4aabbb8dde37d259d342, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731521542453 2024-11-13T18:12:24,683 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d6718fc5c676428ebac4abff9d7c8084, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731521542841 2024-11-13T18:12:24,683 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting fdbeebf9affa4d2daedf97ab6a6a9b8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731521544027 2024-11-13T18:12:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:12:24,692 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:24,693 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/6a848ceac68b4f25a6fc07774aac0e51 is 50, key is test_row_0/C:col10/1731521544144/Put/seqid=0 2024-11-13T18:12:24,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741915_1091 (size=13153) 2024-11-13T18:12:24,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/27b77c5cd61441069c77e511a07b92bd 2024-11-13T18:12:24,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/0e1fe1141b7a45679cd9b4740f9cae99 is 50, key is test_row_0/B:col10/1731521544149/Put/seqid=0 2024-11-13T18:12:24,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741916_1092 (size=12301) 2024-11-13T18:12:24,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:12:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:24,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:24,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:24,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:24,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:12:24,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:24,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:24,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:24,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:24,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:12:24,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521604944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521604945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:24,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521604945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:25,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:12:25,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:25,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:25,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:25,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,113 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/6a848ceac68b4f25a6fc07774aac0e51 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/6a848ceac68b4f25a6fc07774aac0e51 2024-11-13T18:12:25,120 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into 6a848ceac68b4f25a6fc07774aac0e51(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:25,120 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:25,120 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=13, startTime=1731521544229; duration=0sec 2024-11-13T18:12:25,121 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:25,121 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:25,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/0e1fe1141b7a45679cd9b4740f9cae99 2024-11-13T18:12:25,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/94e7ab6c3eea48fa8494b5ebcf534de1 is 50, key is test_row_0/C:col10/1731521544149/Put/seqid=0 2024-11-13T18:12:25,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741917_1093 (size=12301) 2024-11-13T18:12:25,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:12:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:25,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:12:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:12:25,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:25,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:25,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:25,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521605454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:25,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521605454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:25,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521605455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:25,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:12:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:25,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:25,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/94e7ab6c3eea48fa8494b5ebcf534de1 2024-11-13T18:12:25,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/27b77c5cd61441069c77e511a07b92bd as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/27b77c5cd61441069c77e511a07b92bd 2024-11-13T18:12:25,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/27b77c5cd61441069c77e511a07b92bd, entries=200, sequenceid=360, filesize=14.4 K 2024-11-13T18:12:25,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/0e1fe1141b7a45679cd9b4740f9cae99 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0e1fe1141b7a45679cd9b4740f9cae99 2024-11-13T18:12:25,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0e1fe1141b7a45679cd9b4740f9cae99, entries=150, sequenceid=360, filesize=12.0 K 2024-11-13T18:12:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/94e7ab6c3eea48fa8494b5ebcf534de1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/94e7ab6c3eea48fa8494b5ebcf534de1 2024-11-13T18:12:25,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/94e7ab6c3eea48fa8494b5ebcf534de1, entries=150, sequenceid=360, filesize=12.0 K 2024-11-13T18:12:25,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 401e2f643e0c1bcf64dcec1f711ca8ed in 1305ms, sequenceid=360, compaction requested=false 2024-11-13T18:12:25,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:25,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:12:25,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:25,669 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:12:25,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:25,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/31a1dc4d2e194aa5adb96909ff4ecb96 is 50, key is test_row_0/A:col10/1731521544311/Put/seqid=0 2024-11-13T18:12:25,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741918_1094 (size=12301) 2024-11-13T18:12:25,700 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/31a1dc4d2e194aa5adb96909ff4ecb96 2024-11-13T18:12:25,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/2cda5681fd8948f3b2f00953402930fb is 50, key is test_row_0/B:col10/1731521544311/Put/seqid=0 2024-11-13T18:12:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:12:25,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741919_1095 (size=12301) 2024-11-13T18:12:26,143 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/2cda5681fd8948f3b2f00953402930fb 2024-11-13T18:12:26,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/1573f288fae04664b60af58c7bcc4b94 is 50, key is test_row_0/C:col10/1731521544311/Put/seqid=0 2024-11-13T18:12:26,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741920_1096 (size=12301) 2024-11-13T18:12:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:26,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:26,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521606489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:26,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521606490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:26,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521606495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:26,563 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/1573f288fae04664b60af58c7bcc4b94 2024-11-13T18:12:26,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/31a1dc4d2e194aa5adb96909ff4ecb96 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/31a1dc4d2e194aa5adb96909ff4ecb96 2024-11-13T18:12:26,581 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/31a1dc4d2e194aa5adb96909ff4ecb96, entries=150, sequenceid=379, filesize=12.0 K 2024-11-13T18:12:26,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/2cda5681fd8948f3b2f00953402930fb as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/2cda5681fd8948f3b2f00953402930fb 2024-11-13T18:12:26,595 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/2cda5681fd8948f3b2f00953402930fb, entries=150, sequenceid=379, filesize=12.0 K 2024-11-13T18:12:26,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/1573f288fae04664b60af58c7bcc4b94 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1573f288fae04664b60af58c7bcc4b94 2024-11-13T18:12:26,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521606605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521606605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:26,609 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1573f288fae04664b60af58c7bcc4b94, entries=150, sequenceid=379, filesize=12.0 K 2024-11-13T18:12:26,610 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 401e2f643e0c1bcf64dcec1f711ca8ed in 941ms, sequenceid=379, compaction requested=true 2024-11-13T18:12:26,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:26,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:26,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-13T18:12:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-13T18:12:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:26,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:12:26,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:26,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:26,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:26,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:26,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:26,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:26,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-13T18:12:26,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0460 sec 2024-11-13T18:12:26,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.0540 sec 2024-11-13T18:12:26,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/9ceeaedb892c4abb81bbde9e5cb46f9f is 50, key is test_row_0/A:col10/1731521546465/Put/seqid=0 2024-11-13T18:12:26,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741921_1097 (size=14741) 2024-11-13T18:12:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:12:26,725 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:26,726 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-13T18:12:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:12:26,728 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:26,729 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:26,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:26,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521606731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:26,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521606815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:26,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521606815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:12:26,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521606844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:26,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:12:26,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:26,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:26,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:26,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:26,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:12:27,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521607064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:27,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:12:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/9ceeaedb892c4abb81bbde9e5cb46f9f 2024-11-13T18:12:27,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/bf0665edb6f24d3e9282873ab42eb373 is 50, key is test_row_0/B:col10/1731521546465/Put/seqid=0 2024-11-13T18:12:27,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521607134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:27,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521607134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:27,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:27,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741922_1098 (size=12301) 2024-11-13T18:12:27,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:12:27,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:12:27,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521607374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:27,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:27,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:12:27,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:27,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:12:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/bf0665edb6f24d3e9282873ab42eb373 2024-11-13T18:12:27,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/1d2e3ef5e3e7471eae1ff95d2212a354 is 50, key is test_row_0/C:col10/1731521546465/Put/seqid=0 2024-11-13T18:12:27,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741923_1099 (size=12301) 2024-11-13T18:12:27,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521607654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:27,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521607654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:27,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:12:27,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:27,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:12:27,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:27,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:27,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:12:27,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521607884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:27,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:28,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:12:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:28,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:28,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/1d2e3ef5e3e7471eae1ff95d2212a354 2024-11-13T18:12:28,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/9ceeaedb892c4abb81bbde9e5cb46f9f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/9ceeaedb892c4abb81bbde9e5cb46f9f 2024-11-13T18:12:28,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/9ceeaedb892c4abb81bbde9e5cb46f9f, entries=200, sequenceid=396, filesize=14.4 K 2024-11-13T18:12:28,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/bf0665edb6f24d3e9282873ab42eb373 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bf0665edb6f24d3e9282873ab42eb373 2024-11-13T18:12:28,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bf0665edb6f24d3e9282873ab42eb373, entries=150, sequenceid=396, filesize=12.0 K 2024-11-13T18:12:28,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/1d2e3ef5e3e7471eae1ff95d2212a354 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1d2e3ef5e3e7471eae1ff95d2212a354 2024-11-13T18:12:28,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1d2e3ef5e3e7471eae1ff95d2212a354, entries=150, sequenceid=396, filesize=12.0 K 2024-11-13T18:12:28,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 401e2f643e0c1bcf64dcec1f711ca8ed in 1466ms, sequenceid=396, compaction requested=true 2024-11-13T18:12:28,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:28,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:28,093 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:28,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:28,093 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:28,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:28,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:28,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:28,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:28,096 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54936 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:28,096 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:28,096 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:28,097 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7748e071f0794dd2907c785e5997263a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/27b77c5cd61441069c77e511a07b92bd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/31a1dc4d2e194aa5adb96909ff4ecb96, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/9ceeaedb892c4abb81bbde9e5cb46f9f] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=53.6 K 2024-11-13T18:12:28,097 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:28,097 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:28,097 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:28,097 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7748e071f0794dd2907c785e5997263a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731521544027 2024-11-13T18:12:28,097 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6699919de02945bea18133b61c18fa4d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0e1fe1141b7a45679cd9b4740f9cae99, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/2cda5681fd8948f3b2f00953402930fb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bf0665edb6f24d3e9282873ab42eb373] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=48.9 K 2024-11-13T18:12:28,098 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6699919de02945bea18133b61c18fa4d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731521544027 2024-11-13T18:12:28,098 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b77c5cd61441069c77e511a07b92bd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731521544149 2024-11-13T18:12:28,098 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1fe1141b7a45679cd9b4740f9cae99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731521544149 2024-11-13T18:12:28,099 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31a1dc4d2e194aa5adb96909ff4ecb96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731521544293 2024-11-13T18:12:28,099 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cda5681fd8948f3b2f00953402930fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731521544293 2024-11-13T18:12:28,099 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ceeaedb892c4abb81bbde9e5cb46f9f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731521546465 2024-11-13T18:12:28,102 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting bf0665edb6f24d3e9282873ab42eb373, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731521546465 2024-11-13T18:12:28,128 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:28,128 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/91cb520ee5d14718994467fc99f60a88 is 50, key is test_row_0/A:col10/1731521546465/Put/seqid=0 2024-11-13T18:12:28,137 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:28,138 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/ea1ee016324b4dfaa4d4fa8228133667 is 50, key is test_row_0/B:col10/1731521546465/Put/seqid=0 2024-11-13T18:12:28,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:12:28,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:28,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:12:28,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:28,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:28,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741924_1100 (size=13289) 2024-11-13T18:12:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/13c2cd6bf8004715b582688baa68af41 is 50, key is test_row_0/A:col10/1731521546659/Put/seqid=0 2024-11-13T18:12:28,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741925_1101 (size=13289) 2024-11-13T18:12:28,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741926_1102 (size=12301) 2024-11-13T18:12:28,600 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/91cb520ee5d14718994467fc99f60a88 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/91cb520ee5d14718994467fc99f60a88 2024-11-13T18:12:28,607 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into 91cb520ee5d14718994467fc99f60a88(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:28,607 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:28,607 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=12, startTime=1731521548093; duration=0sec 2024-11-13T18:12:28,608 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:28,608 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:28,608 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:28,610 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:28,610 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:28,610 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:28,610 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/6a848ceac68b4f25a6fc07774aac0e51, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/94e7ab6c3eea48fa8494b5ebcf534de1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1573f288fae04664b60af58c7bcc4b94, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1d2e3ef5e3e7471eae1ff95d2212a354] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=48.9 K 2024-11-13T18:12:28,611 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a848ceac68b4f25a6fc07774aac0e51, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731521544027 2024-11-13T18:12:28,612 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94e7ab6c3eea48fa8494b5ebcf534de1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731521544149 2024-11-13T18:12:28,613 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1573f288fae04664b60af58c7bcc4b94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731521544293 2024-11-13T18:12:28,613 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d2e3ef5e3e7471eae1ff95d2212a354, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731521546465 2024-11-13T18:12:28,634 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/ea1ee016324b4dfaa4d4fa8228133667 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ea1ee016324b4dfaa4d4fa8228133667 2024-11-13T18:12:28,643 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:28,644 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/7a041eca97f549e8acb98930f14dbdca is 50, key is test_row_0/C:col10/1731521546465/Put/seqid=0 2024-11-13T18:12:28,652 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into ea1ee016324b4dfaa4d4fa8228133667(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:28,652 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:28,652 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=12, startTime=1731521548093; duration=0sec 2024-11-13T18:12:28,652 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:28,652 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:28,676 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/13c2cd6bf8004715b582688baa68af41 2024-11-13T18:12:28,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. as already flushing 2024-11-13T18:12:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:28,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741927_1103 (size=13289) 2024-11-13T18:12:28,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/bd2c564585ce4979bb5ff10c0521911d is 50, key is test_row_0/B:col10/1731521546659/Put/seqid=0 2024-11-13T18:12:28,700 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/7a041eca97f549e8acb98930f14dbdca as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7a041eca97f549e8acb98930f14dbdca 2024-11-13T18:12:28,715 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into 7a041eca97f549e8acb98930f14dbdca(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:28,715 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:28,715 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=12, startTime=1731521548094; duration=0sec 2024-11-13T18:12:28,715 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:28,715 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:28,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741928_1104 (size=12301) 2024-11-13T18:12:28,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521608755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:28,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521608752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521608864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521608864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:12:28,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521608909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:28,924 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 401e2f643e0c1bcf64dcec1f711ca8ed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:29,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521609074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521609074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:29,120 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/bd2c564585ce4979bb5ff10c0521911d 2024-11-13T18:12:29,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/34406824c4b047dba752d6654331d576 is 50, key is test_row_0/C:col10/1731521546659/Put/seqid=0 2024-11-13T18:12:29,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741929_1105 (size=12301) 2024-11-13T18:12:29,147 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/34406824c4b047dba752d6654331d576 2024-11-13T18:12:29,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/13c2cd6bf8004715b582688baa68af41 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/13c2cd6bf8004715b582688baa68af41 2024-11-13T18:12:29,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/13c2cd6bf8004715b582688baa68af41, entries=150, sequenceid=411, filesize=12.0 K 2024-11-13T18:12:29,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/bd2c564585ce4979bb5ff10c0521911d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd2c564585ce4979bb5ff10c0521911d 2024-11-13T18:12:29,177 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd2c564585ce4979bb5ff10c0521911d, entries=150, sequenceid=411, filesize=12.0 K 2024-11-13T18:12:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/34406824c4b047dba752d6654331d576 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/34406824c4b047dba752d6654331d576 2024-11-13T18:12:29,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/34406824c4b047dba752d6654331d576, entries=150, sequenceid=411, filesize=12.0 K 2024-11-13T18:12:29,186 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 401e2f643e0c1bcf64dcec1f711ca8ed in 1020ms, sequenceid=411, compaction requested=false 2024-11-13T18:12:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-13T18:12:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-13T18:12:29,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T18:12:29,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4590 sec 2024-11-13T18:12:29,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.4660 sec 2024-11-13T18:12:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:29,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:12:29,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:29,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/da8bf32350234e85a9ba28ba11a48006 is 50, key is test_row_0/A:col10/1731521548748/Put/seqid=0 2024-11-13T18:12:29,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521609423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:29,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521609424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:29,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741930_1106 (size=12301) 2024-11-13T18:12:29,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/da8bf32350234e85a9ba28ba11a48006 2024-11-13T18:12:29,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/f5f4a007ed55416fbebacfca497c45cd is 50, key is test_row_0/B:col10/1731521548748/Put/seqid=0 2024-11-13T18:12:29,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741931_1107 (size=12301) 2024-11-13T18:12:29,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/f5f4a007ed55416fbebacfca497c45cd 2024-11-13T18:12:29,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/3445807e367c4b6c822cef84f0348742 is 50, key is test_row_0/C:col10/1731521548748/Put/seqid=0 2024-11-13T18:12:29,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521609534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:29,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521609536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:29,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:29,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741932_1108 (size=12301) 2024-11-13T18:12:29,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521609744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:29,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521609745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:29,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:29,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:29,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/3445807e367c4b6c822cef84f0348742 2024-11-13T18:12:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/da8bf32350234e85a9ba28ba11a48006 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/da8bf32350234e85a9ba28ba11a48006 2024-11-13T18:12:30,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/da8bf32350234e85a9ba28ba11a48006, entries=150, sequenceid=434, filesize=12.0 K 2024-11-13T18:12:30,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/f5f4a007ed55416fbebacfca497c45cd as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f5f4a007ed55416fbebacfca497c45cd 2024-11-13T18:12:30,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f5f4a007ed55416fbebacfca497c45cd, entries=150, sequenceid=434, filesize=12.0 K 2024-11-13T18:12:30,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/3445807e367c4b6c822cef84f0348742 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3445807e367c4b6c822cef84f0348742 2024-11-13T18:12:30,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3445807e367c4b6c822cef84f0348742, entries=150, sequenceid=434, filesize=12.0 K 2024-11-13T18:12:30,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 401e2f643e0c1bcf64dcec1f711ca8ed in 627ms, sequenceid=434, compaction requested=true 2024-11-13T18:12:30,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:30,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:30,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:30,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:30,017 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:30,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:30,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:30,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:12:30,017 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:30,018 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:30,018 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:30,019 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:30,019 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/91cb520ee5d14718994467fc99f60a88, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/13c2cd6bf8004715b582688baa68af41, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/da8bf32350234e85a9ba28ba11a48006] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=37.0 K 2024-11-13T18:12:30,019 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:30,019 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:30,019 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:30,020 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 91cb520ee5d14718994467fc99f60a88, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731521546465 2024-11-13T18:12:30,020 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ea1ee016324b4dfaa4d4fa8228133667, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd2c564585ce4979bb5ff10c0521911d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f5f4a007ed55416fbebacfca497c45cd] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=37.0 K 2024-11-13T18:12:30,020 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 13c2cd6bf8004715b582688baa68af41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731521546659 2024-11-13T18:12:30,021 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea1ee016324b4dfaa4d4fa8228133667, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731521546465 2024-11-13T18:12:30,021 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting da8bf32350234e85a9ba28ba11a48006, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731521548748 2024-11-13T18:12:30,023 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd2c564585ce4979bb5ff10c0521911d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731521546659 2024-11-13T18:12:30,024 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5f4a007ed55416fbebacfca497c45cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731521548748 2024-11-13T18:12:30,052 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#96 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:30,053 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/660d08da87e242dca032a0c65310e5d5 is 50, key is test_row_0/A:col10/1731521548748/Put/seqid=0 2024-11-13T18:12:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:30,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T18:12:30,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:30,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:30,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:30,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:30,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:30,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:30,070 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#97 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:30,071 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/4d5222016a674b789d8627ec17b0acdb is 50, key is test_row_0/B:col10/1731521548748/Put/seqid=0 2024-11-13T18:12:30,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741933_1109 (size=13391) 2024-11-13T18:12:30,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/25ce6fe6dcc5445a998e108c644a0197 is 50, key is test_row_0/A:col10/1731521550066/Put/seqid=0 2024-11-13T18:12:30,093 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/660d08da87e242dca032a0c65310e5d5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/660d08da87e242dca032a0c65310e5d5 2024-11-13T18:12:30,100 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into 660d08da87e242dca032a0c65310e5d5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:30,100 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:30,100 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=13, startTime=1731521550017; duration=0sec 2024-11-13T18:12:30,100 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:30,100 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:30,100 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:30,101 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:30,102 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:30,102 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:30,102 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7a041eca97f549e8acb98930f14dbdca, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/34406824c4b047dba752d6654331d576, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3445807e367c4b6c822cef84f0348742] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=37.0 K 2024-11-13T18:12:30,102 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a041eca97f549e8acb98930f14dbdca, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731521546465 2024-11-13T18:12:30,103 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 34406824c4b047dba752d6654331d576, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731521546659 2024-11-13T18:12:30,103 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 3445807e367c4b6c822cef84f0348742, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731521548748 2024-11-13T18:12:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521610131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521610134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741934_1110 (size=13391) 2024-11-13T18:12:30,143 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:30,143 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/3568384bdc1e437497164a873f36da66 is 50, key is test_row_0/C:col10/1731521548748/Put/seqid=0 2024-11-13T18:12:30,150 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/4d5222016a674b789d8627ec17b0acdb as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/4d5222016a674b789d8627ec17b0acdb 2024-11-13T18:12:30,159 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into 4d5222016a674b789d8627ec17b0acdb(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:30,159 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:30,159 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=13, startTime=1731521550017; duration=0sec 2024-11-13T18:12:30,159 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:30,160 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:30,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741935_1111 (size=14741) 2024-11-13T18:12:30,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741936_1112 (size=13391) 2024-11-13T18:12:30,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521610244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521610244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521610454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521610454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/25ce6fe6dcc5445a998e108c644a0197 2024-11-13T18:12:30,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/c38eae06ccaf499fab423dc1a3f8d0d0 is 50, key is test_row_0/B:col10/1731521550066/Put/seqid=0 2024-11-13T18:12:30,606 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/3568384bdc1e437497164a873f36da66 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3568384bdc1e437497164a873f36da66 2024-11-13T18:12:30,617 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into 3568384bdc1e437497164a873f36da66(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:30,617 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:30,617 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=13, startTime=1731521550017; duration=0sec 2024-11-13T18:12:30,617 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:30,618 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:30,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741937_1113 (size=12301) 2024-11-13T18:12:30,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/c38eae06ccaf499fab423dc1a3f8d0d0 2024-11-13T18:12:30,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/231b3e5ea4044d39a21e6ccf793dc326 is 50, key is test_row_0/C:col10/1731521550066/Put/seqid=0 2024-11-13T18:12:30,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741938_1114 (size=12301) 2024-11-13T18:12:30,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/231b3e5ea4044d39a21e6ccf793dc326 2024-11-13T18:12:30,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/25ce6fe6dcc5445a998e108c644a0197 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/25ce6fe6dcc5445a998e108c644a0197 2024-11-13T18:12:30,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/25ce6fe6dcc5445a998e108c644a0197, entries=200, sequenceid=448, filesize=14.4 K 2024-11-13T18:12:30,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/c38eae06ccaf499fab423dc1a3f8d0d0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/c38eae06ccaf499fab423dc1a3f8d0d0 2024-11-13T18:12:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521610765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/c38eae06ccaf499fab423dc1a3f8d0d0, entries=150, sequenceid=448, filesize=12.0 K 2024-11-13T18:12:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521610769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/231b3e5ea4044d39a21e6ccf793dc326 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/231b3e5ea4044d39a21e6ccf793dc326 2024-11-13T18:12:30,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/231b3e5ea4044d39a21e6ccf793dc326, entries=150, sequenceid=448, filesize=12.0 K 2024-11-13T18:12:30,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 401e2f643e0c1bcf64dcec1f711ca8ed in 729ms, sequenceid=448, compaction requested=false 2024-11-13T18:12:30,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:30,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:12:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:30,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/38c3eb119f2b46b28c2fa39b9f35756a is 50, key is test_row_0/A:col10/1731521550095/Put/seqid=0 2024-11-13T18:12:30,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521610870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,887 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-13T18:12:30,887 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:30,889 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-13T18:12:30,889 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:30,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521610878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,894 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:30,895 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:30,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:12:30,896 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:30,897 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:30,897 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:30,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:30,897 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-13T18:12:30,897 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:30,898 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:30,898 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:30,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:30,909 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-13T18:12:30,910 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:30,910 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:30,910 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:30,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:30,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741939_1115 (size=14741) 2024-11-13T18:12:30,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43610 deadline: 1731521610934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,936 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 401e2f643e0c1bcf64dcec1f711ca8ed of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521610984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:30,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521610994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:30,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:30,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:31,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521611195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:31,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521611204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:31,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43600 deadline: 1731521611284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:31,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1731521611295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:31,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/38c3eb119f2b46b28c2fa39b9f35756a 2024-11-13T18:12:31,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/ce51616f7de940bcb567d15dae89f2f2 is 50, key is test_row_0/B:col10/1731521550095/Put/seqid=0 2024-11-13T18:12:31,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741940_1116 (size=12301) 2024-11-13T18:12:31,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521611504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:31,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521611514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:31,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/ce51616f7de940bcb567d15dae89f2f2 2024-11-13T18:12:31,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/ee61017cbf5142b6bc41d2e746f513a2 is 50, key is test_row_0/C:col10/1731521550095/Put/seqid=0 2024-11-13T18:12:31,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741941_1117 (size=12301) 2024-11-13T18:12:32,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:32,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43560 deadline: 1731521612014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:32,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43632 deadline: 1731521612025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:12:32,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/ee61017cbf5142b6bc41d2e746f513a2 2024-11-13T18:12:32,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/38c3eb119f2b46b28c2fa39b9f35756a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/38c3eb119f2b46b28c2fa39b9f35756a 2024-11-13T18:12:32,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/38c3eb119f2b46b28c2fa39b9f35756a, entries=200, sequenceid=470, filesize=14.4 K 2024-11-13T18:12:32,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/ce51616f7de940bcb567d15dae89f2f2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ce51616f7de940bcb567d15dae89f2f2 2024-11-13T18:12:32,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ce51616f7de940bcb567d15dae89f2f2, entries=150, sequenceid=470, filesize=12.0 K 2024-11-13T18:12:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/ee61017cbf5142b6bc41d2e746f513a2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ee61017cbf5142b6bc41d2e746f513a2 2024-11-13T18:12:32,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ee61017cbf5142b6bc41d2e746f513a2, entries=150, sequenceid=470, filesize=12.0 K 2024-11-13T18:12:32,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 401e2f643e0c1bcf64dcec1f711ca8ed in 1380ms, sequenceid=470, compaction requested=true 2024-11-13T18:12:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:32,196 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:32,196 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:32,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:32,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 401e2f643e0c1bcf64dcec1f711ca8ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:32,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:32,198 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:32,198 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/A is initiating minor compaction (all files) 2024-11-13T18:12:32,198 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/A in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:32,198 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/660d08da87e242dca032a0c65310e5d5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/25ce6fe6dcc5445a998e108c644a0197, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/38c3eb119f2b46b28c2fa39b9f35756a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=41.9 K 2024-11-13T18:12:32,198 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:32,198 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/B is initiating minor compaction (all files) 2024-11-13T18:12:32,198 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/B in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:32,199 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/4d5222016a674b789d8627ec17b0acdb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/c38eae06ccaf499fab423dc1a3f8d0d0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ce51616f7de940bcb567d15dae89f2f2] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=37.1 K 2024-11-13T18:12:32,199 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 660d08da87e242dca032a0c65310e5d5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731521548748 2024-11-13T18:12:32,199 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d5222016a674b789d8627ec17b0acdb, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731521548748 2024-11-13T18:12:32,200 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25ce6fe6dcc5445a998e108c644a0197, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731521549412 2024-11-13T18:12:32,200 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c38eae06ccaf499fab423dc1a3f8d0d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731521549412 2024-11-13T18:12:32,200 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c3eb119f2b46b28c2fa39b9f35756a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731521550095 2024-11-13T18:12:32,200 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ce51616f7de940bcb567d15dae89f2f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731521550095 2024-11-13T18:12:32,222 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#A#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:32,222 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/cf19a320ae8d4185bd8f4d8a517e9c10 is 50, key is test_row_0/A:col10/1731521550095/Put/seqid=0 2024-11-13T18:12:32,238 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:32,239 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/fe66acfc7e6745308f91c5b600d9531d is 50, key is test_row_0/B:col10/1731521550095/Put/seqid=0 2024-11-13T18:12:32,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741942_1118 (size=13493) 2024-11-13T18:12:32,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741943_1119 (size=13493) 2024-11-13T18:12:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:32,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:12:32,296 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-13T18:12:32,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:32,296 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:32,296 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:32,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:32,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:32,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:32,297 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:32,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:32,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:32,297 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/fe66acfc7e6745308f91c5b600d9531d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/fe66acfc7e6745308f91c5b600d9531d 2024-11-13T18:12:32,297 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:32,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/5043d5f15c874bc0909fe3ea589f98ba is 50, key is test_row_0/A:col10/1731521550868/Put/seqid=0 2024-11-13T18:12:32,303 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/B of 401e2f643e0c1bcf64dcec1f711ca8ed into fe66acfc7e6745308f91c5b600d9531d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:32,303 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:32,303 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/B, priority=13, startTime=1731521552196; duration=0sec 2024-11-13T18:12:32,304 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:32,304 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:B 2024-11-13T18:12:32,304 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:32,306 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-13T18:12:32,306 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:32,306 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:32,306 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:32,306 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:32,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:32,306 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 401e2f643e0c1bcf64dcec1f711ca8ed/C is initiating minor compaction (all files) 2024-11-13T18:12:32,307 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 401e2f643e0c1bcf64dcec1f711ca8ed/C in TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:32,307 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3568384bdc1e437497164a873f36da66, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/231b3e5ea4044d39a21e6ccf793dc326, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ee61017cbf5142b6bc41d2e746f513a2] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp, totalSize=37.1 K 2024-11-13T18:12:32,307 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 3568384bdc1e437497164a873f36da66, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731521548748 2024-11-13T18:12:32,308 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 231b3e5ea4044d39a21e6ccf793dc326, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731521549412 2024-11-13T18:12:32,308 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ee61017cbf5142b6bc41d2e746f513a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731521550095 2024-11-13T18:12:32,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741944_1120 (size=12301) 2024-11-13T18:12:32,327 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 401e2f643e0c1bcf64dcec1f711ca8ed#C#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:32,327 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/2a288004331e45448bf10324b66c1eb0 is 50, key is test_row_0/C:col10/1731521550095/Put/seqid=0 2024-11-13T18:12:32,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741945_1121 (size=13493) 2024-11-13T18:12:32,678 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/cf19a320ae8d4185bd8f4d8a517e9c10 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/cf19a320ae8d4185bd8f4d8a517e9c10 2024-11-13T18:12:32,696 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/A of 401e2f643e0c1bcf64dcec1f711ca8ed into cf19a320ae8d4185bd8f4d8a517e9c10(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:32,696 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:32,696 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/A, priority=13, startTime=1731521552196; duration=0sec 2024-11-13T18:12:32,697 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:32,697 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:A 2024-11-13T18:12:32,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/5043d5f15c874bc0909fe3ea589f98ba 2024-11-13T18:12:32,744 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/2a288004331e45448bf10324b66c1eb0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/2a288004331e45448bf10324b66c1eb0 2024-11-13T18:12:32,756 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 401e2f643e0c1bcf64dcec1f711ca8ed/C of 401e2f643e0c1bcf64dcec1f711ca8ed into 2a288004331e45448bf10324b66c1eb0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:32,756 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:32,756 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., storeName=401e2f643e0c1bcf64dcec1f711ca8ed/C, priority=13, startTime=1731521552197; duration=0sec 2024-11-13T18:12:32,756 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:32,756 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 401e2f643e0c1bcf64dcec1f711ca8ed:C 2024-11-13T18:12:32,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/6c70f9497a0240c6b86637d45cb3d53f is 50, key is test_row_0/B:col10/1731521550868/Put/seqid=0 2024-11-13T18:12:32,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741946_1122 (size=12301) 2024-11-13T18:12:32,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/6c70f9497a0240c6b86637d45cb3d53f 2024-11-13T18:12:32,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/4aff8784dac34cf483790ab9e68cb672 is 50, key is test_row_0/C:col10/1731521550868/Put/seqid=0 2024-11-13T18:12:32,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741947_1123 (size=12301) 2024-11-13T18:12:33,036 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-13T18:12:33,036 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:33,036 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:33,037 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:33,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:33,039 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-13T18:12:33,039 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:33,039 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:33,039 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:33,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:33,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/4aff8784dac34cf483790ab9e68cb672 2024-11-13T18:12:33,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/5043d5f15c874bc0909fe3ea589f98ba as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/5043d5f15c874bc0909fe3ea589f98ba 2024-11-13T18:12:33,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/5043d5f15c874bc0909fe3ea589f98ba, entries=150, sequenceid=485, filesize=12.0 K 2024-11-13T18:12:33,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/6c70f9497a0240c6b86637d45cb3d53f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6c70f9497a0240c6b86637d45cb3d53f 2024-11-13T18:12:33,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6c70f9497a0240c6b86637d45cb3d53f, entries=150, sequenceid=485, filesize=12.0 K 2024-11-13T18:12:33,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/4aff8784dac34cf483790ab9e68cb672 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/4aff8784dac34cf483790ab9e68cb672 2024-11-13T18:12:33,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/4aff8784dac34cf483790ab9e68cb672, entries=150, sequenceid=485, filesize=12.0 K 2024-11-13T18:12:33,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for 401e2f643e0c1bcf64dcec1f711ca8ed in 965ms, sequenceid=485, compaction requested=false 2024-11-13T18:12:33,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 401e2f643e0c1bcf64dcec1f711ca8ed: 2024-11-13T18:12:34,965 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-13T18:12:34,966 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:34,966 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:34,966 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:34,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:34,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:12:34,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-13T18:12:34,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2500 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2465 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1113 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3329 rows 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1118 2024-11-13T18:12:34,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3346 rows 2024-11-13T18:12:34,967 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:12:34,967 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:12:34,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:34,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:34,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:34,972 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:12:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:12:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:12:34,983 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521554983"}]},"ts":"1731521554983"} 2024-11-13T18:12:34,983 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:12:34,987 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:12:34,987 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:12:34,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:12:34,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=401e2f643e0c1bcf64dcec1f711ca8ed, UNASSIGN}] 2024-11-13T18:12:34,995 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=401e2f643e0c1bcf64dcec1f711ca8ed, UNASSIGN 2024-11-13T18:12:34,997 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=401e2f643e0c1bcf64dcec1f711ca8ed, regionState=CLOSING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:34,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=401e2f643e0c1bcf64dcec1f711ca8ed, UNASSIGN because future has completed 2024-11-13T18:12:35,000 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:12:35,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:12:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:12:35,158 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:35,158 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:12:35,158 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing 401e2f643e0c1bcf64dcec1f711ca8ed, disabling compactions & flushes 2024-11-13T18:12:35,159 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:35,159 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:35,159 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. after waiting 0 ms 2024-11-13T18:12:35,159 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:35,159 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(2902): Flushing 401e2f643e0c1bcf64dcec1f711ca8ed 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T18:12:35,159 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=A 2024-11-13T18:12:35,159 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:35,159 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=B 2024-11-13T18:12:35,160 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:35,160 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 401e2f643e0c1bcf64dcec1f711ca8ed, store=C 2024-11-13T18:12:35,160 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:35,164 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f941c439e0a44527a035e528e42ecfe6 is 50, key is test_row_0/A:col10/1731521554964/Put/seqid=0 2024-11-13T18:12:35,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741948_1124 (size=12301) 2024-11-13T18:12:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:12:35,572 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f941c439e0a44527a035e528e42ecfe6 2024-11-13T18:12:35,580 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/f2aae411aa994fa6a70d47fa0afd46d5 is 50, key is test_row_0/B:col10/1731521554964/Put/seqid=0 2024-11-13T18:12:35,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741949_1125 (size=12301) 2024-11-13T18:12:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:12:35,985 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/f2aae411aa994fa6a70d47fa0afd46d5 2024-11-13T18:12:35,999 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/a6a7c170b3d047ada37ad04d34024a21 is 50, key is test_row_0/C:col10/1731521554964/Put/seqid=0 2024-11-13T18:12:36,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741950_1126 (size=12301) 2024-11-13T18:12:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:12:36,419 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/a6a7c170b3d047ada37ad04d34024a21 2024-11-13T18:12:36,427 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/A/f941c439e0a44527a035e528e42ecfe6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f941c439e0a44527a035e528e42ecfe6 2024-11-13T18:12:36,433 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f941c439e0a44527a035e528e42ecfe6, entries=150, sequenceid=495, filesize=12.0 K 2024-11-13T18:12:36,434 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/B/f2aae411aa994fa6a70d47fa0afd46d5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f2aae411aa994fa6a70d47fa0afd46d5 2024-11-13T18:12:36,439 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f2aae411aa994fa6a70d47fa0afd46d5, entries=150, sequenceid=495, filesize=12.0 K 2024-11-13T18:12:36,440 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/.tmp/C/a6a7c170b3d047ada37ad04d34024a21 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/a6a7c170b3d047ada37ad04d34024a21 2024-11-13T18:12:36,445 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/a6a7c170b3d047ada37ad04d34024a21, entries=150, sequenceid=495, filesize=12.0 K 2024-11-13T18:12:36,446 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 401e2f643e0c1bcf64dcec1f711ca8ed in 1287ms, sequenceid=495, compaction requested=true 2024-11-13T18:12:36,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/de7aa9cfcad543e782f24ef75477c7b8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f35dd5ef95344e888e00731538499a89, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/bf1f660d96f949e3b29504c58bd374b1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f6ade35c4abf4794ac3c1879ecbd3796, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c702b6df845548609f63775d4566fdbf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b0b49b243de14fbb9c19ef47f3338bc4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/e22c761ca82f4b7da229b82b9b4619e1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f1ab116c889d484a8e5a120e1ce11109, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/56715e6f6879474cae96c159e5f98f57, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c0438db69b564224812bcd7a27815626, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/10fc49c45083426db1c096b7e23778f2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/5a9a300d168c4d9ba20750e11f59e9c6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/6536d7ad3e2e48ef9e05fdcf6a42829e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/e76b97b27cc740ad9867c4f1ea92c238, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/aa1065fe1cc94cfeb880b6c4fc38ad52, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/58c54be8ca864a9b90ca315895ed4f88, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/67548bb64bc048169fed135f956260e8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7fe52c34dad44b879273566215cc83a0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b4e5010b5f0e483fabcc186fc473229f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/29e49d0056bd4df5a6ea5768a4f353e5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c9d1a36eb59b483589b7704ecfe03091, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/0e5df77be6f245b693274942e28d8ab1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/3cf9620aa7054318bcb0fce928a67c55, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/553200f7fcb5414cb24b1808f93a3a38, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/57e8746ddb0b4b4db2d239aa2a5515d1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7748e071f0794dd2907c785e5997263a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/27b77c5cd61441069c77e511a07b92bd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/31a1dc4d2e194aa5adb96909ff4ecb96, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/9ceeaedb892c4abb81bbde9e5cb46f9f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/91cb520ee5d14718994467fc99f60a88, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/13c2cd6bf8004715b582688baa68af41, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/660d08da87e242dca032a0c65310e5d5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/da8bf32350234e85a9ba28ba11a48006, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/25ce6fe6dcc5445a998e108c644a0197, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/38c3eb119f2b46b28c2fa39b9f35756a] to archive 2024-11-13T18:12:36,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:12:36,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/de7aa9cfcad543e782f24ef75477c7b8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/de7aa9cfcad543e782f24ef75477c7b8 2024-11-13T18:12:36,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f35dd5ef95344e888e00731538499a89 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f35dd5ef95344e888e00731538499a89 2024-11-13T18:12:36,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/bf1f660d96f949e3b29504c58bd374b1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/bf1f660d96f949e3b29504c58bd374b1 2024-11-13T18:12:36,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f6ade35c4abf4794ac3c1879ecbd3796 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f6ade35c4abf4794ac3c1879ecbd3796 2024-11-13T18:12:36,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c702b6df845548609f63775d4566fdbf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c702b6df845548609f63775d4566fdbf 2024-11-13T18:12:36,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b0b49b243de14fbb9c19ef47f3338bc4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b0b49b243de14fbb9c19ef47f3338bc4 2024-11-13T18:12:36,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/e22c761ca82f4b7da229b82b9b4619e1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/e22c761ca82f4b7da229b82b9b4619e1 2024-11-13T18:12:36,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f1ab116c889d484a8e5a120e1ce11109 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f1ab116c889d484a8e5a120e1ce11109 2024-11-13T18:12:36,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/56715e6f6879474cae96c159e5f98f57 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/56715e6f6879474cae96c159e5f98f57 2024-11-13T18:12:36,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c0438db69b564224812bcd7a27815626 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c0438db69b564224812bcd7a27815626 2024-11-13T18:12:36,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/10fc49c45083426db1c096b7e23778f2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/10fc49c45083426db1c096b7e23778f2 2024-11-13T18:12:36,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/5a9a300d168c4d9ba20750e11f59e9c6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/5a9a300d168c4d9ba20750e11f59e9c6 2024-11-13T18:12:36,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/6536d7ad3e2e48ef9e05fdcf6a42829e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/6536d7ad3e2e48ef9e05fdcf6a42829e 2024-11-13T18:12:36,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/e76b97b27cc740ad9867c4f1ea92c238 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/e76b97b27cc740ad9867c4f1ea92c238 2024-11-13T18:12:36,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/aa1065fe1cc94cfeb880b6c4fc38ad52 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/aa1065fe1cc94cfeb880b6c4fc38ad52 2024-11-13T18:12:36,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/58c54be8ca864a9b90ca315895ed4f88 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/58c54be8ca864a9b90ca315895ed4f88 2024-11-13T18:12:36,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/67548bb64bc048169fed135f956260e8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/67548bb64bc048169fed135f956260e8 2024-11-13T18:12:36,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7fe52c34dad44b879273566215cc83a0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7fe52c34dad44b879273566215cc83a0 2024-11-13T18:12:36,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b4e5010b5f0e483fabcc186fc473229f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/b4e5010b5f0e483fabcc186fc473229f 2024-11-13T18:12:36,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/29e49d0056bd4df5a6ea5768a4f353e5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/29e49d0056bd4df5a6ea5768a4f353e5 2024-11-13T18:12:36,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c9d1a36eb59b483589b7704ecfe03091 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/c9d1a36eb59b483589b7704ecfe03091 2024-11-13T18:12:36,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/0e5df77be6f245b693274942e28d8ab1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/0e5df77be6f245b693274942e28d8ab1 2024-11-13T18:12:36,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/3cf9620aa7054318bcb0fce928a67c55 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/3cf9620aa7054318bcb0fce928a67c55 2024-11-13T18:12:36,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/553200f7fcb5414cb24b1808f93a3a38 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/553200f7fcb5414cb24b1808f93a3a38 2024-11-13T18:12:36,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/57e8746ddb0b4b4db2d239aa2a5515d1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/57e8746ddb0b4b4db2d239aa2a5515d1 2024-11-13T18:12:36,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7748e071f0794dd2907c785e5997263a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/7748e071f0794dd2907c785e5997263a 2024-11-13T18:12:36,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/27b77c5cd61441069c77e511a07b92bd to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/27b77c5cd61441069c77e511a07b92bd 2024-11-13T18:12:36,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/31a1dc4d2e194aa5adb96909ff4ecb96 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/31a1dc4d2e194aa5adb96909ff4ecb96 2024-11-13T18:12:36,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/9ceeaedb892c4abb81bbde9e5cb46f9f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/9ceeaedb892c4abb81bbde9e5cb46f9f 2024-11-13T18:12:36,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/91cb520ee5d14718994467fc99f60a88 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/91cb520ee5d14718994467fc99f60a88 2024-11-13T18:12:36,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/13c2cd6bf8004715b582688baa68af41 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/13c2cd6bf8004715b582688baa68af41 2024-11-13T18:12:36,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/660d08da87e242dca032a0c65310e5d5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/660d08da87e242dca032a0c65310e5d5 2024-11-13T18:12:36,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/da8bf32350234e85a9ba28ba11a48006 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/da8bf32350234e85a9ba28ba11a48006 2024-11-13T18:12:36,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/25ce6fe6dcc5445a998e108c644a0197 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/25ce6fe6dcc5445a998e108c644a0197 2024-11-13T18:12:36,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/38c3eb119f2b46b28c2fa39b9f35756a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/38c3eb119f2b46b28c2fa39b9f35756a 2024-11-13T18:12:36,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd75442033e646ee9fa21e2af11b096d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0d9873147ee2455abb8d1a6435da5d4f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6f950db1699a4f589f5615b4a2e988df, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/33c724ff4278495dafded55d03e25f81, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ead6a663e8114ccd946279cf6f599ef3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e4d208e6af0e4db6b839f7f087df1a78, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/d01d5a9464724516bc39dd8e19274935, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/a0dc95ec9d6f4f53b0d5061c0d8c74e2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/9475d10e6fb546bfa684621887056504, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/18a4441a07dc43b797bdf403a4a93316, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/492be3abc0bc4d15ac876db9f053c69a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e6086a2b39c042ef906395059cbbbc24, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ec298fc400a44daa9e2ab0435268c07c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/83e9752c9082467abe23b2ad4b300a43, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/fabff8fba00844499680550e2b03b86a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/21acf727509440d28da0106125b48c59, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f4357dd7388540ea9d07cf83702c1389, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/44aeefc419f54f5194a40eb615f7fc70, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e761641aa9094beeb26ca34d133bacb7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/46bd0934500a4c4ebd90350c254f93fd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/20e37c193b454664bf160f9208f57863, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/56c3427bbd61492e97f120de22340664, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/54b54c3b3d4645d19224ffe367b27a2e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/370963797e694597b0fc199b6b3467cc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6699919de02945bea18133b61c18fa4d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/85bcb2d508ee44258072a37dbcd907a7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0e1fe1141b7a45679cd9b4740f9cae99, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/2cda5681fd8948f3b2f00953402930fb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ea1ee016324b4dfaa4d4fa8228133667, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bf0665edb6f24d3e9282873ab42eb373, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd2c564585ce4979bb5ff10c0521911d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/4d5222016a674b789d8627ec17b0acdb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f5f4a007ed55416fbebacfca497c45cd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/c38eae06ccaf499fab423dc1a3f8d0d0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ce51616f7de940bcb567d15dae89f2f2] to archive 2024-11-13T18:12:36,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:12:36,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd75442033e646ee9fa21e2af11b096d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd75442033e646ee9fa21e2af11b096d 2024-11-13T18:12:36,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0d9873147ee2455abb8d1a6435da5d4f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0d9873147ee2455abb8d1a6435da5d4f 2024-11-13T18:12:36,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6f950db1699a4f589f5615b4a2e988df to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6f950db1699a4f589f5615b4a2e988df 2024-11-13T18:12:36,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/33c724ff4278495dafded55d03e25f81 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/33c724ff4278495dafded55d03e25f81 2024-11-13T18:12:36,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ead6a663e8114ccd946279cf6f599ef3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ead6a663e8114ccd946279cf6f599ef3 2024-11-13T18:12:36,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e4d208e6af0e4db6b839f7f087df1a78 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e4d208e6af0e4db6b839f7f087df1a78 2024-11-13T18:12:36,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/d01d5a9464724516bc39dd8e19274935 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/d01d5a9464724516bc39dd8e19274935 2024-11-13T18:12:36,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/a0dc95ec9d6f4f53b0d5061c0d8c74e2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/a0dc95ec9d6f4f53b0d5061c0d8c74e2 2024-11-13T18:12:36,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/9475d10e6fb546bfa684621887056504 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/9475d10e6fb546bfa684621887056504 2024-11-13T18:12:36,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/18a4441a07dc43b797bdf403a4a93316 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/18a4441a07dc43b797bdf403a4a93316 2024-11-13T18:12:36,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/492be3abc0bc4d15ac876db9f053c69a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/492be3abc0bc4d15ac876db9f053c69a 2024-11-13T18:12:36,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e6086a2b39c042ef906395059cbbbc24 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e6086a2b39c042ef906395059cbbbc24 2024-11-13T18:12:36,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ec298fc400a44daa9e2ab0435268c07c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ec298fc400a44daa9e2ab0435268c07c 2024-11-13T18:12:36,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/83e9752c9082467abe23b2ad4b300a43 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/83e9752c9082467abe23b2ad4b300a43 2024-11-13T18:12:36,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/fabff8fba00844499680550e2b03b86a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/fabff8fba00844499680550e2b03b86a 2024-11-13T18:12:36,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/21acf727509440d28da0106125b48c59 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/21acf727509440d28da0106125b48c59 2024-11-13T18:12:36,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f4357dd7388540ea9d07cf83702c1389 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f4357dd7388540ea9d07cf83702c1389 2024-11-13T18:12:36,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/44aeefc419f54f5194a40eb615f7fc70 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/44aeefc419f54f5194a40eb615f7fc70 2024-11-13T18:12:36,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e761641aa9094beeb26ca34d133bacb7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/e761641aa9094beeb26ca34d133bacb7 2024-11-13T18:12:36,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/46bd0934500a4c4ebd90350c254f93fd to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/46bd0934500a4c4ebd90350c254f93fd 2024-11-13T18:12:36,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/20e37c193b454664bf160f9208f57863 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/20e37c193b454664bf160f9208f57863 2024-11-13T18:12:36,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/56c3427bbd61492e97f120de22340664 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/56c3427bbd61492e97f120de22340664 2024-11-13T18:12:36,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/54b54c3b3d4645d19224ffe367b27a2e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/54b54c3b3d4645d19224ffe367b27a2e 2024-11-13T18:12:36,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/370963797e694597b0fc199b6b3467cc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/370963797e694597b0fc199b6b3467cc 2024-11-13T18:12:36,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6699919de02945bea18133b61c18fa4d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6699919de02945bea18133b61c18fa4d 2024-11-13T18:12:36,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/85bcb2d508ee44258072a37dbcd907a7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/85bcb2d508ee44258072a37dbcd907a7 2024-11-13T18:12:36,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0e1fe1141b7a45679cd9b4740f9cae99 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/0e1fe1141b7a45679cd9b4740f9cae99 2024-11-13T18:12:36,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/2cda5681fd8948f3b2f00953402930fb to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/2cda5681fd8948f3b2f00953402930fb 2024-11-13T18:12:36,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ea1ee016324b4dfaa4d4fa8228133667 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ea1ee016324b4dfaa4d4fa8228133667 2024-11-13T18:12:36,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bf0665edb6f24d3e9282873ab42eb373 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bf0665edb6f24d3e9282873ab42eb373 2024-11-13T18:12:36,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd2c564585ce4979bb5ff10c0521911d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/bd2c564585ce4979bb5ff10c0521911d 2024-11-13T18:12:36,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/4d5222016a674b789d8627ec17b0acdb to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/4d5222016a674b789d8627ec17b0acdb 2024-11-13T18:12:36,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f5f4a007ed55416fbebacfca497c45cd to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f5f4a007ed55416fbebacfca497c45cd 2024-11-13T18:12:36,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/c38eae06ccaf499fab423dc1a3f8d0d0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/c38eae06ccaf499fab423dc1a3f8d0d0 2024-11-13T18:12:36,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ce51616f7de940bcb567d15dae89f2f2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/ce51616f7de940bcb567d15dae89f2f2 2024-11-13T18:12:36,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/bb2dd0ecb35d46b79cb8e20f8078f49e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/c17f29ed65fa49f0a2bb592885a47a95, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cf122331869145f79fe413b15910c473, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/b2b687d651324c5e8df4ce2aa9a401af, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/934c4a8127bc465baf3b3f7d5887dc82, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/85a8a06540e94559b23f4f44c82af65b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3f887aa7095748be8e346c3771d9e2e9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/4963938e6cc1452e90993cba3f57f2b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/6bdcb84f1f3445849aa8e4f66cc0ad4d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7343d392c9cf400fa5824c637b3d35ab, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/337d41868af4474c9d93c21a9736d94a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/acd6219a60d84949ab5d3bf6e1b892e2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/30b10a3cf00b410db7072952ac98f8fe, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/341a59ec48144fd696c5c7d5c73086d8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5bf1f707ea304d6db72745af4deff7cd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/661296b2669248178a0793366c1711d4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cb84c32c65304abaabe2ad026f1e41ef, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/39d8e35b02f94a7eb28601cee8da52f0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5215f66c9301476db14c52c2d69a2604, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd88f9db32fe4243af4a2381b87b2330, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ca7b3b930a5d46ea8d853d006e3ebf21, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/f12f675ddd6d4aabbb8dde37d259d342, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd6e1ca50b8a4e829bcc09ec88525f60, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/d6718fc5c676428ebac4abff9d7c8084, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/6a848ceac68b4f25a6fc07774aac0e51, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fdbeebf9affa4d2daedf97ab6a6a9b8e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/94e7ab6c3eea48fa8494b5ebcf534de1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1573f288fae04664b60af58c7bcc4b94, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7a041eca97f549e8acb98930f14dbdca, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1d2e3ef5e3e7471eae1ff95d2212a354, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/34406824c4b047dba752d6654331d576, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3568384bdc1e437497164a873f36da66, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3445807e367c4b6c822cef84f0348742, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/231b3e5ea4044d39a21e6ccf793dc326, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ee61017cbf5142b6bc41d2e746f513a2] to archive 2024-11-13T18:12:36,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:12:36,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/bb2dd0ecb35d46b79cb8e20f8078f49e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/bb2dd0ecb35d46b79cb8e20f8078f49e 2024-11-13T18:12:36,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/c17f29ed65fa49f0a2bb592885a47a95 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/c17f29ed65fa49f0a2bb592885a47a95 2024-11-13T18:12:36,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cf122331869145f79fe413b15910c473 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cf122331869145f79fe413b15910c473 2024-11-13T18:12:36,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/b2b687d651324c5e8df4ce2aa9a401af to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/b2b687d651324c5e8df4ce2aa9a401af 2024-11-13T18:12:36,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/934c4a8127bc465baf3b3f7d5887dc82 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/934c4a8127bc465baf3b3f7d5887dc82 2024-11-13T18:12:36,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/85a8a06540e94559b23f4f44c82af65b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/85a8a06540e94559b23f4f44c82af65b 2024-11-13T18:12:36,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3f887aa7095748be8e346c3771d9e2e9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3f887aa7095748be8e346c3771d9e2e9 2024-11-13T18:12:36,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/4963938e6cc1452e90993cba3f57f2b5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/4963938e6cc1452e90993cba3f57f2b5 2024-11-13T18:12:36,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/6bdcb84f1f3445849aa8e4f66cc0ad4d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/6bdcb84f1f3445849aa8e4f66cc0ad4d 2024-11-13T18:12:36,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7343d392c9cf400fa5824c637b3d35ab to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7343d392c9cf400fa5824c637b3d35ab 2024-11-13T18:12:36,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/337d41868af4474c9d93c21a9736d94a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/337d41868af4474c9d93c21a9736d94a 2024-11-13T18:12:36,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/acd6219a60d84949ab5d3bf6e1b892e2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/acd6219a60d84949ab5d3bf6e1b892e2 2024-11-13T18:12:36,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/30b10a3cf00b410db7072952ac98f8fe to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/30b10a3cf00b410db7072952ac98f8fe 2024-11-13T18:12:36,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/341a59ec48144fd696c5c7d5c73086d8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/341a59ec48144fd696c5c7d5c73086d8 2024-11-13T18:12:36,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5bf1f707ea304d6db72745af4deff7cd to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5bf1f707ea304d6db72745af4deff7cd 2024-11-13T18:12:36,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/661296b2669248178a0793366c1711d4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/661296b2669248178a0793366c1711d4 2024-11-13T18:12:36,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cb84c32c65304abaabe2ad026f1e41ef to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/cb84c32c65304abaabe2ad026f1e41ef 2024-11-13T18:12:36,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/39d8e35b02f94a7eb28601cee8da52f0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/39d8e35b02f94a7eb28601cee8da52f0 2024-11-13T18:12:36,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5215f66c9301476db14c52c2d69a2604 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/5215f66c9301476db14c52c2d69a2604 2024-11-13T18:12:36,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd88f9db32fe4243af4a2381b87b2330 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd88f9db32fe4243af4a2381b87b2330 2024-11-13T18:12:36,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ca7b3b930a5d46ea8d853d006e3ebf21 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ca7b3b930a5d46ea8d853d006e3ebf21 2024-11-13T18:12:36,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/f12f675ddd6d4aabbb8dde37d259d342 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/f12f675ddd6d4aabbb8dde37d259d342 2024-11-13T18:12:36,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd6e1ca50b8a4e829bcc09ec88525f60 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fd6e1ca50b8a4e829bcc09ec88525f60 2024-11-13T18:12:36,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/d6718fc5c676428ebac4abff9d7c8084 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/d6718fc5c676428ebac4abff9d7c8084 2024-11-13T18:12:36,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/6a848ceac68b4f25a6fc07774aac0e51 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/6a848ceac68b4f25a6fc07774aac0e51 2024-11-13T18:12:36,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fdbeebf9affa4d2daedf97ab6a6a9b8e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/fdbeebf9affa4d2daedf97ab6a6a9b8e 2024-11-13T18:12:36,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/94e7ab6c3eea48fa8494b5ebcf534de1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/94e7ab6c3eea48fa8494b5ebcf534de1 2024-11-13T18:12:36,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1573f288fae04664b60af58c7bcc4b94 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1573f288fae04664b60af58c7bcc4b94 2024-11-13T18:12:36,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7a041eca97f549e8acb98930f14dbdca to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/7a041eca97f549e8acb98930f14dbdca 2024-11-13T18:12:36,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1d2e3ef5e3e7471eae1ff95d2212a354 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/1d2e3ef5e3e7471eae1ff95d2212a354 2024-11-13T18:12:36,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/34406824c4b047dba752d6654331d576 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/34406824c4b047dba752d6654331d576 2024-11-13T18:12:36,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3568384bdc1e437497164a873f36da66 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3568384bdc1e437497164a873f36da66 2024-11-13T18:12:36,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3445807e367c4b6c822cef84f0348742 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/3445807e367c4b6c822cef84f0348742 2024-11-13T18:12:36,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/231b3e5ea4044d39a21e6ccf793dc326 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/231b3e5ea4044d39a21e6ccf793dc326 2024-11-13T18:12:36,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ee61017cbf5142b6bc41d2e746f513a2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/ee61017cbf5142b6bc41d2e746f513a2 2024-11-13T18:12:36,684 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/recovered.edits/498.seqid, newMaxSeqId=498, maxSeqId=1 2024-11-13T18:12:36,685 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. 2024-11-13T18:12:36,685 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for 401e2f643e0c1bcf64dcec1f711ca8ed: Waiting for close lock at 1731521555158Running coprocessor pre-close hooks at 1731521555158Disabling compacts and flushes for region at 1731521555158Disabling writes for close at 1731521555159 (+1 ms)Obtaining lock to block concurrent updates at 1731521555159Preparing flush snapshotting stores in 401e2f643e0c1bcf64dcec1f711ca8ed at 1731521555159Finished memstore snapshotting TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731521555160 (+1 ms)Flushing stores of TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed. at 1731521555160Flushing 401e2f643e0c1bcf64dcec1f711ca8ed/A: creating writer at 1731521555160Flushing 401e2f643e0c1bcf64dcec1f711ca8ed/A: appending metadata at 1731521555164 (+4 ms)Flushing 401e2f643e0c1bcf64dcec1f711ca8ed/A: closing flushed file at 1731521555164Flushing 401e2f643e0c1bcf64dcec1f711ca8ed/B: creating writer at 1731521555576 (+412 ms)Flushing 401e2f643e0c1bcf64dcec1f711ca8ed/B: appending metadata at 1731521555579 (+3 ms)Flushing 401e2f643e0c1bcf64dcec1f711ca8ed/B: closing flushed file at 1731521555579Flushing 401e2f643e0c1bcf64dcec1f711ca8ed/C: creating writer at 1731521555992 (+413 ms)Flushing 401e2f643e0c1bcf64dcec1f711ca8ed/C: appending metadata at 1731521555998 (+6 ms)Flushing 401e2f643e0c1bcf64dcec1f711ca8ed/C: closing flushed file at 1731521555998Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a47ac6a: reopening flushed file at 1731521556426 (+428 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15ef7e7e: reopening flushed file at 1731521556433 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d921173: reopening flushed file at 1731521556439 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 401e2f643e0c1bcf64dcec1f711ca8ed in 1287ms, sequenceid=495, compaction requested=true at 1731521556446 (+7 ms)Writing region close event to WAL at 1731521556679 (+233 ms)Running coprocessor post-close hooks at 1731521556684 (+5 ms)Closed at 1731521556685 (+1 ms) 2024-11-13T18:12:36,687 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed 401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:36,688 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=401e2f643e0c1bcf64dcec1f711ca8ed, regionState=CLOSED 2024-11-13T18:12:36,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:12:36,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-13T18:12:36,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure 401e2f643e0c1bcf64dcec1f711ca8ed, server=fda96cf025dc,44483,1731521526892 in 1.6930 sec 2024-11-13T18:12:36,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-13T18:12:36,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=401e2f643e0c1bcf64dcec1f711ca8ed, UNASSIGN in 1.7010 sec 2024-11-13T18:12:36,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T18:12:36,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7090 sec 2024-11-13T18:12:36,702 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521556702"}]},"ts":"1731521556702"} 2024-11-13T18:12:36,704 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:12:36,704 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:12:36,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7290 sec 2024-11-13T18:12:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:12:37,135 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:37,140 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:12:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:12:37,149 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:12:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:12:37,151 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:12:37,156 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:37,161 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/recovered.edits] 2024-11-13T18:12:37,167 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/5043d5f15c874bc0909fe3ea589f98ba to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/5043d5f15c874bc0909fe3ea589f98ba 2024-11-13T18:12:37,170 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/cf19a320ae8d4185bd8f4d8a517e9c10 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/cf19a320ae8d4185bd8f4d8a517e9c10 2024-11-13T18:12:37,172 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f941c439e0a44527a035e528e42ecfe6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/A/f941c439e0a44527a035e528e42ecfe6 2024-11-13T18:12:37,175 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6c70f9497a0240c6b86637d45cb3d53f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/6c70f9497a0240c6b86637d45cb3d53f 2024-11-13T18:12:37,177 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f2aae411aa994fa6a70d47fa0afd46d5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/f2aae411aa994fa6a70d47fa0afd46d5 2024-11-13T18:12:37,180 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/fe66acfc7e6745308f91c5b600d9531d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/B/fe66acfc7e6745308f91c5b600d9531d 2024-11-13T18:12:37,187 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/2a288004331e45448bf10324b66c1eb0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/2a288004331e45448bf10324b66c1eb0 2024-11-13T18:12:37,189 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/4aff8784dac34cf483790ab9e68cb672 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/4aff8784dac34cf483790ab9e68cb672 2024-11-13T18:12:37,192 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/a6a7c170b3d047ada37ad04d34024a21 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/C/a6a7c170b3d047ada37ad04d34024a21 2024-11-13T18:12:37,201 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/recovered.edits/498.seqid to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed/recovered.edits/498.seqid 2024-11-13T18:12:37,202 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/401e2f643e0c1bcf64dcec1f711ca8ed 2024-11-13T18:12:37,202 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:12:37,207 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:12:37,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-13T18:12:37,221 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:12:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:12:37,285 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:12:37,289 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:12:37,289 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:12:37,289 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731521557289"}]},"ts":"9223372036854775807"} 2024-11-13T18:12:37,298 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:12:37,298 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 401e2f643e0c1bcf64dcec1f711ca8ed, NAME => 'TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:12:37,298 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:12:37,299 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731521557298"}]},"ts":"9223372036854775807"} 2024-11-13T18:12:37,302 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:12:37,304 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:12:37,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 163 msec 2024-11-13T18:12:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:12:37,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:12:37,470 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:37,484 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 210) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fda96cf025dc:44483-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-873439638_22 at /127.0.0.1:35030 [Waiting for operation #266] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1307743371_22 at /127.0.0.1:40454 [Waiting for operation #192] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1307743371_22 at /127.0.0.1:35130 [Waiting for operation #253] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/java.lang.StackTraceElement.initStackTraceElements(Native Method) java.base@17.0.11/java.lang.StackTraceElement.of(StackTraceElement.java:541) java.base@17.0.11/java.lang.Throwable.getOurStackTrace(Throwable.java:839) java.base@17.0.11/java.lang.Throwable.getStackTrace(Throwable.java:831) app//io.opentelemetry.context.StrictContextStorage.attach(StrictContextStorage.java:68) app//io.opentelemetry.context.Context.makeCurrent(Context.java:202) app//io.opentelemetry.context.ImplicitContextKeyed.makeCurrent(ImplicitContextKeyed.java:33) app//org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:186) app//org.apache.hadoop.hbase.client.AsyncRegionLocator.clearCache(AsyncRegionLocator.java:218) app//org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$DeleteTableProcedureBiConsumer.onFinished(RawAsyncHBaseAdmin.java:2804) app//org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$ProcedureBiConsumer.accept(RawAsyncHBaseAdmin.java:2701) app//org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$ProcedureBiConsumer.accept(RawAsyncHBaseAdmin.java:2689) app//org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) app//org.apache.hadoop.hbase.util.FutureUtils$$Lambda$431/0x00007ff3509c17d0.accept(Unknown Source) java.base@17.0.11/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) java.base@17.0.11/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) java.base@17.0.11/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) java.base@17.0.11/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) app//org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin.lambda$getProcedureResult$262(RawAsyncHBaseAdmin.java:3050) app//org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$$Lambda$857/0x00007ff350bbdc50.accept(Unknown Source) app//org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) app//org.apache.hadoop.hbase.util.FutureUtils$$Lambda$431/0x00007ff3509c17d0.accept(Unknown Source) java.base@17.0.11/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) java.base@17.0.11/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) java.base@17.0.11/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) java.base@17.0.11/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) app//org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:80) app//org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller$$Lambda$842/0x00007ff350baeda0.accept(Unknown Source) app//org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) app//org.apache.hadoop.hbase.util.FutureUtils$$Lambda$431/0x00007ff3509c17d0.accept(Unknown Source) java.base@17.0.11/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) java.base@17.0.11/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) java.base@17.0.11/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) java.base@17.0.11/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) app//org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:468) app//org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) app//org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) app//org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) app//org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) app//org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) app//org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) app//org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=677 (was 660) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1026 (was 260) - AvailableMemoryMB LEAK? - 2024-11-13T18:12:37,502 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=677, ProcessCount=11, AvailableMemoryMB=1025 2024-11-13T18:12:37,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:12:37,508 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:12:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:12:37,512 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:12:37,512 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:37,513 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:12:37,514 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-11-13T18:12:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:12:37,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741951_1127 (size=1061) 2024-11-13T18:12:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:12:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:12:37,942 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1 2024-11-13T18:12:37,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741952_1128 (size=53) 2024-11-13T18:12:37,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:12:37,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 634afcda8bfed22b98df3e6e31daa427, disabling compactions & flushes 2024-11-13T18:12:37,957 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:37,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:37,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. after waiting 0 ms 2024-11-13T18:12:37,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:37,957 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:37,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 634afcda8bfed22b98df3e6e31daa427: Waiting for close lock at 1731521557957Disabling compacts and flushes for region at 1731521557957Disabling writes for close at 1731521557957Writing region close event to WAL at 1731521557957Closed at 1731521557957 2024-11-13T18:12:37,959 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:12:37,959 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731521557959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521557959"}]},"ts":"1731521557959"} 2024-11-13T18:12:37,962 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:12:37,964 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:12:37,964 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521557964"}]},"ts":"1731521557964"} 2024-11-13T18:12:37,968 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:12:37,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, ASSIGN}] 2024-11-13T18:12:37,970 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, ASSIGN 2024-11-13T18:12:37,971 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, ASSIGN; state=OFFLINE, location=fda96cf025dc,44483,1731521526892; forceNewPlan=false, retain=false 2024-11-13T18:12:38,122 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=634afcda8bfed22b98df3e6e31daa427, regionState=OPENING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:38,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, ASSIGN because future has completed 2024-11-13T18:12:38,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:12:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:12:38,303 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:38,303 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:12:38,303 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,303 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:12:38,303 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,303 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,305 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,308 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:12:38,310 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634afcda8bfed22b98df3e6e31daa427 columnFamilyName A 2024-11-13T18:12:38,310 DEBUG [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:38,311 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(327): Store=634afcda8bfed22b98df3e6e31daa427/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:38,312 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,322 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:12:38,323 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634afcda8bfed22b98df3e6e31daa427 columnFamilyName B 2024-11-13T18:12:38,323 DEBUG [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:38,323 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(327): Store=634afcda8bfed22b98df3e6e31daa427/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:38,324 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,327 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:12:38,327 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634afcda8bfed22b98df3e6e31daa427 columnFamilyName C 2024-11-13T18:12:38,327 DEBUG [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:38,328 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(327): Store=634afcda8bfed22b98df3e6e31daa427/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:38,328 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:38,328 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,329 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,330 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,332 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,332 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,333 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:12:38,335 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,351 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:12:38,354 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened 634afcda8bfed22b98df3e6e31daa427; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67203219, jitterRate=0.0014059990644454956}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:12:38,354 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,356 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for 634afcda8bfed22b98df3e6e31daa427: Running coprocessor pre-open hook at 1731521558304Writing region info on filesystem at 1731521558304Initializing all the Stores at 1731521558304Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521558305 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521558305Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521558305Cleaning up temporary data from old regions at 1731521558332 (+27 ms)Running coprocessor post-open hooks at 1731521558354 (+22 ms)Region opened successfully at 1731521558356 (+2 ms) 2024-11-13T18:12:38,358 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., pid=26, masterSystemTime=1731521558285 2024-11-13T18:12:38,361 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:38,361 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:38,370 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=634afcda8bfed22b98df3e6e31daa427, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:38,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:12:38,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T18:12:38,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 in 252 msec 2024-11-13T18:12:38,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-13T18:12:38,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, ASSIGN in 420 msec 2024-11-13T18:12:38,392 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:12:38,392 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521558392"}]},"ts":"1731521558392"} 2024-11-13T18:12:38,394 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:12:38,395 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:12:38,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 887 msec 2024-11-13T18:12:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:12:38,655 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:38,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a0c1161, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:38,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:38,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:38,669 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71bb314e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:38,673 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:38,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37421322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:38,675 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:38,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:38,681 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:12:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,37165,1731521525941 2024-11-13T18:12:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c72d130 2024-11-13T18:12:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:12:38,684 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:12:38,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:12:38,701 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:12:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T18:12:38,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741953_1129 (size=1097) 2024-11-13T18:12:38,752 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T18:12:38,752 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T18:12:38,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:12:38,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, REOPEN/MOVE}] 2024-11-13T18:12:38,769 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, REOPEN/MOVE 2024-11-13T18:12:38,770 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=634afcda8bfed22b98df3e6e31daa427, regionState=CLOSING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:38,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, REOPEN/MOVE because future has completed 2024-11-13T18:12:38,773 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:12:38,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:12:38,927 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,927 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:12:38,928 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 634afcda8bfed22b98df3e6e31daa427, disabling compactions & flushes 2024-11-13T18:12:38,928 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:38,928 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:38,928 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. after waiting 0 ms 2024-11-13T18:12:38,928 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:38,932 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:12:38,933 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:38,933 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 634afcda8bfed22b98df3e6e31daa427: Waiting for close lock at 1731521558927Running coprocessor pre-close hooks at 1731521558927Disabling compacts and flushes for region at 1731521558927Disabling writes for close at 1731521558928 (+1 ms)Writing region close event to WAL at 1731521558928Running coprocessor post-close hooks at 1731521558933 (+5 ms)Closed at 1731521558933 2024-11-13T18:12:38,933 WARN [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: 634afcda8bfed22b98df3e6e31daa427 to self. 2024-11-13T18:12:38,935 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:38,936 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=634afcda8bfed22b98df3e6e31daa427, regionState=CLOSED 2024-11-13T18:12:38,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:12:38,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T18:12:38,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 in 165 msec 2024-11-13T18:12:38,941 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, REOPEN/MOVE; state=CLOSED, location=fda96cf025dc,44483,1731521526892; forceNewPlan=false, retain=true 2024-11-13T18:12:39,092 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=634afcda8bfed22b98df3e6e31daa427, regionState=OPENING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, REOPEN/MOVE because future has completed 2024-11-13T18:12:39,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:12:39,250 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:39,250 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:12:39,250 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,251 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:12:39,251 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,251 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,256 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,257 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:12:39,263 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634afcda8bfed22b98df3e6e31daa427 columnFamilyName A 2024-11-13T18:12:39,266 DEBUG [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:39,266 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(327): Store=634afcda8bfed22b98df3e6e31daa427/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:39,267 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,268 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:12:39,268 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634afcda8bfed22b98df3e6e31daa427 columnFamilyName B 2024-11-13T18:12:39,268 DEBUG [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:39,268 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(327): Store=634afcda8bfed22b98df3e6e31daa427/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:39,268 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,269 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:12:39,269 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634afcda8bfed22b98df3e6e31daa427 columnFamilyName C 2024-11-13T18:12:39,269 DEBUG [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:39,270 INFO [StoreOpener-634afcda8bfed22b98df3e6e31daa427-1 {}] regionserver.HStore(327): Store=634afcda8bfed22b98df3e6e31daa427/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:12:39,270 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:39,270 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,271 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,272 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,273 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,273 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,274 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:12:39,275 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,276 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 634afcda8bfed22b98df3e6e31daa427; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60114921, jitterRate=-0.10421787202358246}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:12:39,276 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,277 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 634afcda8bfed22b98df3e6e31daa427: Running coprocessor pre-open hook at 1731521559251Writing region info on filesystem at 1731521559251Initializing all the Stores at 1731521559252 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521559252Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521559252Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521559252Cleaning up temporary data from old regions at 1731521559273 (+21 ms)Running coprocessor post-open hooks at 1731521559276 (+3 ms)Region opened successfully at 1731521559277 (+1 ms) 2024-11-13T18:12:39,278 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., pid=31, masterSystemTime=1731521559246 2024-11-13T18:12:39,280 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:39,280 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:39,280 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=634afcda8bfed22b98df3e6e31daa427, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:12:39,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-13T18:12:39,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 in 189 msec 2024-11-13T18:12:39,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-13T18:12:39,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, REOPEN/MOVE in 518 msec 2024-11-13T18:12:39,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-13T18:12:39,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-11-13T18:12:39,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 586 msec 2024-11-13T18:12:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T18:12:39,297 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:39,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fbbb1a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:39,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:39,300 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:39,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:39,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:39,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d0ea814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:39,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:39,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:39,302 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554d2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce47c6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:39,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:39,305 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:39,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:39,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:39,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1389b68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:39,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc1fe9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587a908e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:39,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:39,309 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44716f63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:39,311 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3fe248, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdd1e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:39,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:39,314 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@282d69b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:39,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7bd923, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a8b2d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:39,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:39,322 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a180aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:39,324 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4274fb1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10af1c47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:39,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:39,330 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9682e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:39,332 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:39,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd76749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58feca65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:39,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:39,334 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0cd909, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:39,335 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580c75eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@789be709, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,339 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:39,339 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:39,340 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e90d2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:39,342 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:39,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1701504c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7239c052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:12:39,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:12:39,346 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:12:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:12:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:12:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59abe598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:12:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:12:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:39,347 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:12:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:12:39,349 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:39,349 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:39,349 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:39,350 DEBUG [Thread-653 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:39,353 DEBUG [Thread-653 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:39,354 DEBUG [Thread-645 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:39,354 DEBUG [Thread-645 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:39,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:39,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:39,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:39,360 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:12:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:12:39,362 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:39,363 DEBUG [Thread-647 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:39,363 DEBUG [Thread-647 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:12:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:39,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:39,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:12:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:12:39,372 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:39,373 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:39,374 DEBUG [Thread-651 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:39,374 DEBUG [Thread-651 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:39,375 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:39,375 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:39,375 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:39,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:39,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:12:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-13T18:12:39,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:39,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:12:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:12:39,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:39,382 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:12:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:12:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:12:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:12:39,383 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:39,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:12:39,384 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:39,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:12:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:12:39,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:39,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:39,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:39,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:39,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:39,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:39,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137a537026819c466e80044f0c1aff972b_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521559379/Put/seqid=0 2024-11-13T18:12:39,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521619456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521559432, max timestamp=1731521559460Num uniques 859; 2024-11-13T18:12:39,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:12:39,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521619462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521559432, max timestamp=1731521559460Num uniques 859; 2024-11-13T18:12:39,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:12:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:12:39,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521559432, max timestamp=1731521559460Num uniques 859; 2024-11-13T18:12:39,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:12:39,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741954_1130 (size=17034) 2024-11-13T18:12:39,499 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:39,505 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137a537026819c466e80044f0c1aff972b_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137a537026819c466e80044f0c1aff972b_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,506 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/02eed6c2dec245fbb2648c43b362405d, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:39,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/02eed6c2dec245fbb2648c43b362405d is 175, key is test_row_0/A:col10/1731521559379/Put/seqid=0 2024-11-13T18:12:39,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741955_1131 (size=48139) 2024-11-13T18:12:39,537 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/02eed6c2dec245fbb2648c43b362405d 2024-11-13T18:12:39,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:39,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:39,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:39,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521619574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521619574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521619576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521619578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521619581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/5eb76bf35afd4443bb23219c56744fc6 is 50, key is test_row_0/B:col10/1731521559379/Put/seqid=0 2024-11-13T18:12:39,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741956_1132 (size=12001) 2024-11-13T18:12:39,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/5eb76bf35afd4443bb23219c56744fc6 2024-11-13T18:12:39,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:39,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:39,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:39,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:39,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:39,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:12:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:39,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/e1b5e9ebe75a4502b6fee84d5bdf6096 is 50, key is test_row_0/C:col10/1731521559379/Put/seqid=0 2024-11-13T18:12:39,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521619695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521619695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521619695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741957_1133 (size=12001) 2024-11-13T18:12:39,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/e1b5e9ebe75a4502b6fee84d5bdf6096 2024-11-13T18:12:39,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/02eed6c2dec245fbb2648c43b362405d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/02eed6c2dec245fbb2648c43b362405d 2024-11-13T18:12:39,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521619794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/02eed6c2dec245fbb2648c43b362405d, entries=250, sequenceid=16, filesize=47.0 K 2024-11-13T18:12:39,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521619797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:39,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:39,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:39,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/5eb76bf35afd4443bb23219c56744fc6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5eb76bf35afd4443bb23219c56744fc6 2024-11-13T18:12:39,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5eb76bf35afd4443bb23219c56744fc6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T18:12:39,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/e1b5e9ebe75a4502b6fee84d5bdf6096 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e1b5e9ebe75a4502b6fee84d5bdf6096 2024-11-13T18:12:39,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:39,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:39,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:39,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:39,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:39,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:39,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e1b5e9ebe75a4502b6fee84d5bdf6096, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T18:12:39,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 634afcda8bfed22b98df3e6e31daa427 in 495ms, sequenceid=16, compaction requested=false 2024-11-13T18:12:39,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:39,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-13T18:12:39,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:39,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:39,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:39,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:39,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:39,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:39,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132545b68df6c043f8aea8bb4e7dfb4c50_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521559935/Put/seqid=0 2024-11-13T18:12:40,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:40,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:40,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:12:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741958_1134 (size=14594) 2024-11-13T18:12:40,050 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521620027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521620053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,082 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132545b68df6c043f8aea8bb4e7dfb4c50_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132545b68df6c043f8aea8bb4e7dfb4c50_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:40,092 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/a691946e1c80495181a55e0a3104a2e9, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:40,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/a691946e1c80495181a55e0a3104a2e9 is 175, key is test_row_0/A:col10/1731521559935/Put/seqid=0 2024-11-13T18:12:40,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521620072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521620114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521620124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741959_1135 (size=39549) 2024-11-13T18:12:40,141 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/a691946e1c80495181a55e0a3104a2e9 2024-11-13T18:12:40,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:40,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521620164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/4b61681d381c413bbce43ce0be12bb36 is 50, key is test_row_0/B:col10/1731521559935/Put/seqid=0 2024-11-13T18:12:40,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521620184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521620205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741960_1136 (size=12001) 2024-11-13T18:12:40,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/4b61681d381c413bbce43ce0be12bb36 2024-11-13T18:12:40,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/fa0d6c6059cc4a3cb8d831fc302d49c8 is 50, key is test_row_0/C:col10/1731521559935/Put/seqid=0 2024-11-13T18:12:40,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741961_1137 (size=12001) 2024-11-13T18:12:40,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/fa0d6c6059cc4a3cb8d831fc302d49c8 2024-11-13T18:12:40,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/a691946e1c80495181a55e0a3104a2e9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a691946e1c80495181a55e0a3104a2e9 2024-11-13T18:12:40,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a691946e1c80495181a55e0a3104a2e9, entries=200, sequenceid=52, filesize=38.6 K 2024-11-13T18:12:40,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/4b61681d381c413bbce43ce0be12bb36 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4b61681d381c413bbce43ce0be12bb36 2024-11-13T18:12:40,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4b61681d381c413bbce43ce0be12bb36, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T18:12:40,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/fa0d6c6059cc4a3cb8d831fc302d49c8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fa0d6c6059cc4a3cb8d831fc302d49c8 2024-11-13T18:12:40,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:40,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:40,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fa0d6c6059cc4a3cb8d831fc302d49c8, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T18:12:40,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 634afcda8bfed22b98df3e6e31daa427 in 374ms, sequenceid=52, compaction requested=false 2024-11-13T18:12:40,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:12:40,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:40,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:40,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:40,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:40,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:40,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:40,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:40,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133fa9170a5418412ba81a5ce5044b0f8c_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_1/A:col10/1731521560016/Put/seqid=0 2024-11-13T18:12:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:12:40,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741962_1138 (size=12154) 2024-11-13T18:12:40,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521620519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521620520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521620521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:40,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:40,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521620624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521620634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521620644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521620645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521620645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:40,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521620844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521620864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521620864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:40,926 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:40,932 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133fa9170a5418412ba81a5ce5044b0f8c_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133fa9170a5418412ba81a5ce5044b0f8c_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:40,933 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/5cc7d5f95854439bb1e14252e503d977, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:40,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/5cc7d5f95854439bb1e14252e503d977 is 175, key is test_row_1/A:col10/1731521560016/Put/seqid=0 2024-11-13T18:12:40,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:40,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:40,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:40,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:40,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741963_1139 (size=30951) 2024-11-13T18:12:41,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521621154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521621185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521621185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:41,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:41,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,400 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/5cc7d5f95854439bb1e14252e503d977 2024-11-13T18:12:41,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:41,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:41,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/2ccbeacb88df45f5aabdaac768fc81c6 is 50, key is test_row_1/B:col10/1731521560016/Put/seqid=0 2024-11-13T18:12:41,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741964_1140 (size=9657) 2024-11-13T18:12:41,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/2ccbeacb88df45f5aabdaac768fc81c6 2024-11-13T18:12:41,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/b96a2ecb6e39476c9f85c2ef0d7c61dc is 50, key is test_row_1/C:col10/1731521560016/Put/seqid=0 2024-11-13T18:12:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:12:41,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741965_1141 (size=9657) 2024-11-13T18:12:41,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/b96a2ecb6e39476c9f85c2ef0d7c61dc 2024-11-13T18:12:41,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/5cc7d5f95854439bb1e14252e503d977 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5cc7d5f95854439bb1e14252e503d977 2024-11-13T18:12:41,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:41,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5cc7d5f95854439bb1e14252e503d977, entries=150, sequenceid=65, filesize=30.2 K 2024-11-13T18:12:41,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:41,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/2ccbeacb88df45f5aabdaac768fc81c6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2ccbeacb88df45f5aabdaac768fc81c6 2024-11-13T18:12:41,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2ccbeacb88df45f5aabdaac768fc81c6, entries=100, sequenceid=65, filesize=9.4 K 2024-11-13T18:12:41,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/b96a2ecb6e39476c9f85c2ef0d7c61dc as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/b96a2ecb6e39476c9f85c2ef0d7c61dc 2024-11-13T18:12:41,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/b96a2ecb6e39476c9f85c2ef0d7c61dc, entries=100, sequenceid=65, filesize=9.4 K 2024-11-13T18:12:41,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 634afcda8bfed22b98df3e6e31daa427 in 1157ms, sequenceid=65, compaction requested=true 2024-11-13T18:12:41,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:41,602 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:41,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:41,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:41,602 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:41,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:41,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:41,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:41,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:41,603 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:41,603 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/B is initiating minor compaction (all files) 2024-11-13T18:12:41,604 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/B in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,604 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5eb76bf35afd4443bb23219c56744fc6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4b61681d381c413bbce43ce0be12bb36, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2ccbeacb88df45f5aabdaac768fc81c6] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=32.9 K 2024-11-13T18:12:41,604 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:41,604 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/A is initiating minor compaction (all files) 2024-11-13T18:12:41,604 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/A in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,604 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/02eed6c2dec245fbb2648c43b362405d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a691946e1c80495181a55e0a3104a2e9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5cc7d5f95854439bb1e14252e503d977] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=115.9 K 2024-11-13T18:12:41,604 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eb76bf35afd4443bb23219c56744fc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731521559379 2024-11-13T18:12:41,604 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,604 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/02eed6c2dec245fbb2648c43b362405d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a691946e1c80495181a55e0a3104a2e9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5cc7d5f95854439bb1e14252e503d977] 2024-11-13T18:12:41,605 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b61681d381c413bbce43ce0be12bb36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731521559570 2024-11-13T18:12:41,606 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ccbeacb88df45f5aabdaac768fc81c6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731521560016 2024-11-13T18:12:41,607 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02eed6c2dec245fbb2648c43b362405d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731521559370 2024-11-13T18:12:41,607 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a691946e1c80495181a55e0a3104a2e9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731521559570 2024-11-13T18:12:41,608 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cc7d5f95854439bb1e14252e503d977, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731521560016 2024-11-13T18:12:41,631 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:41,641 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:41,642 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/34593026383f4f9c91b340cefd7b9150 is 50, key is test_row_0/B:col10/1731521559935/Put/seqid=0 2024-11-13T18:12:41,658 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113fe181a1e825a475db252e5b1362ba471_634afcda8bfed22b98df3e6e31daa427 store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:41,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:12:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:41,663 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113fe181a1e825a475db252e5b1362ba471_634afcda8bfed22b98df3e6e31daa427, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:41,664 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fe181a1e825a475db252e5b1362ba471_634afcda8bfed22b98df3e6e31daa427 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:41,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741966_1142 (size=12104) 2024-11-13T18:12:41,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521621703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521621704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521621704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a4b4cae8bb834f3f952170644c1a6c5a_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521560518/Put/seqid=0 2024-11-13T18:12:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521621705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,719 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/34593026383f4f9c91b340cefd7b9150 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/34593026383f4f9c91b340cefd7b9150 2024-11-13T18:12:41,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521621713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,726 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/B of 634afcda8bfed22b98df3e6e31daa427 into 34593026383f4f9c91b340cefd7b9150(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:41,726 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:41,726 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/B, priority=13, startTime=1731521561602; duration=0sec 2024-11-13T18:12:41,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:41,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741967_1143 (size=4469) 2024-11-13T18:12:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,728 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:41,728 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:B 2024-11-13T18:12:41,728 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:41,729 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#A#compaction#123 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,729 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/889a7e7d5c574fafa7ba7c2c68ed03ce is 175, key is test_row_0/A:col10/1731521559935/Put/seqid=0 2024-11-13T18:12:41,737 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:41,737 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/C is initiating minor compaction (all files) 2024-11-13T18:12:41,737 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/C in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,738 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e1b5e9ebe75a4502b6fee84d5bdf6096, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fa0d6c6059cc4a3cb8d831fc302d49c8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/b96a2ecb6e39476c9f85c2ef0d7c61dc] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=32.9 K 2024-11-13T18:12:41,739 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e1b5e9ebe75a4502b6fee84d5bdf6096, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731521559379 2024-11-13T18:12:41,741 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting fa0d6c6059cc4a3cb8d831fc302d49c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731521559570 2024-11-13T18:12:41,742 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b96a2ecb6e39476c9f85c2ef0d7c61dc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731521560016 2024-11-13T18:12:41,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741968_1144 (size=14594) 2024-11-13T18:12:41,749 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:41,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a4b4cae8bb834f3f952170644c1a6c5a_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4b4cae8bb834f3f952170644c1a6c5a_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:41,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/ee6892e17bbd4a75a3b76ef502e15cfa, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:41,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/ee6892e17bbd4a75a3b76ef502e15cfa is 175, key is test_row_0/A:col10/1731521560518/Put/seqid=0 2024-11-13T18:12:41,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741969_1145 (size=31165) 2024-11-13T18:12:41,810 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#C#compaction#126 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:41,811 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/457aeb6039744b3183060bf4abfa489f is 50, key is test_row_0/C:col10/1731521559935/Put/seqid=0 2024-11-13T18:12:41,817 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/889a7e7d5c574fafa7ba7c2c68ed03ce as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/889a7e7d5c574fafa7ba7c2c68ed03ce 2024-11-13T18:12:41,825 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/A of 634afcda8bfed22b98df3e6e31daa427 into 889a7e7d5c574fafa7ba7c2c68ed03ce(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:41,825 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:41,825 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/A, priority=13, startTime=1731521561601; duration=0sec 2024-11-13T18:12:41,825 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:41,825 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:A 2024-11-13T18:12:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521621824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521621824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521621824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:41,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741970_1146 (size=39549) 2024-11-13T18:12:41,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:41,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:41,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:41,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:41,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741971_1147 (size=12104) 2024-11-13T18:12:41,909 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/457aeb6039744b3183060bf4abfa489f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/457aeb6039744b3183060bf4abfa489f 2024-11-13T18:12:41,919 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/C of 634afcda8bfed22b98df3e6e31daa427 into 457aeb6039744b3183060bf4abfa489f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:41,919 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:41,919 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/C, priority=13, startTime=1731521561602; duration=0sec 2024-11-13T18:12:41,919 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:41,919 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:C 2024-11-13T18:12:42,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521622044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521622054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521622054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:42,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:42,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:42,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,243 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:12:42,265 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/ee6892e17bbd4a75a3b76ef502e15cfa 2024-11-13T18:12:42,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/4c4e54e8f9cc4d8aaf5b15b4f5a09e8b is 50, key is test_row_0/B:col10/1731521560518/Put/seqid=0 2024-11-13T18:12:42,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:42,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:42,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741972_1148 (size=12001) 2024-11-13T18:12:42,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521622363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:42,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521622364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521622364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:42,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521622726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,736 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 634afcda8bfed22b98df3e6e31daa427 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:42,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521622726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:42,738 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_1' in 634afcda8bfed22b98df3e6e31daa427 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:42,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/4c4e54e8f9cc4d8aaf5b15b4f5a09e8b 2024-11-13T18:12:42,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/28d4a095c82c4e02b264f560f57d5ef2 is 50, key is test_row_0/C:col10/1731521560518/Put/seqid=0 2024-11-13T18:12:42,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741973_1149 (size=12001) 2024-11-13T18:12:42,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/28d4a095c82c4e02b264f560f57d5ef2 2024-11-13T18:12:42,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:42,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:42,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/ee6892e17bbd4a75a3b76ef502e15cfa as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/ee6892e17bbd4a75a3b76ef502e15cfa 2024-11-13T18:12:42,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/ee6892e17bbd4a75a3b76ef502e15cfa, entries=200, sequenceid=85, filesize=38.6 K 2024-11-13T18:12:42,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/4c4e54e8f9cc4d8aaf5b15b4f5a09e8b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4c4e54e8f9cc4d8aaf5b15b4f5a09e8b 2024-11-13T18:12:42,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4c4e54e8f9cc4d8aaf5b15b4f5a09e8b, entries=150, sequenceid=85, filesize=11.7 K 2024-11-13T18:12:42,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/28d4a095c82c4e02b264f560f57d5ef2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/28d4a095c82c4e02b264f560f57d5ef2 2024-11-13T18:12:42,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/28d4a095c82c4e02b264f560f57d5ef2, entries=150, sequenceid=85, filesize=11.7 K 2024-11-13T18:12:42,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 634afcda8bfed22b98df3e6e31daa427 in 1198ms, sequenceid=85, compaction requested=false 2024-11-13T18:12:42,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:42,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:12:42,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:42,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:42,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:42,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:42,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:42,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:42,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111304c394a5e2f246f386b4dfb4a84afeb2_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521561698/Put/seqid=0 2024-11-13T18:12:42,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741974_1150 (size=12154) 2024-11-13T18:12:42,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:42,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:42,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:42,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521622958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:42,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521622964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521622965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:43,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521623084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521623096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:43,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521623096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:43,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:43,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:43,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:43,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:43,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521623315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521623315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:43,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521623316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:43,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:43,361 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:43,371 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111304c394a5e2f246f386b4dfb4a84afeb2_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304c394a5e2f246f386b4dfb4a84afeb2_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:43,373 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/1c642e46e4c94a578bf5a97ceba34a3b, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:43,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/1c642e46e4c94a578bf5a97ceba34a3b is 175, key is test_row_0/A:col10/1731521561698/Put/seqid=0 2024-11-13T18:12:43,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741975_1151 (size=30955) 2024-11-13T18:12:43,384 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/1c642e46e4c94a578bf5a97ceba34a3b 2024-11-13T18:12:43,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/d939f2ca07fd49429138b090b2329edf is 50, key is test_row_0/B:col10/1731521561698/Put/seqid=0 2024-11-13T18:12:43,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741976_1152 (size=12001) 2024-11-13T18:12:43,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/d939f2ca07fd49429138b090b2329edf 2024-11-13T18:12:43,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/d505c863518b4c93b5974fffc5c2381a is 50, key is test_row_0/C:col10/1731521561698/Put/seqid=0 2024-11-13T18:12:43,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741977_1153 (size=12001) 2024-11-13T18:12:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:12:43,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521623635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:43,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521623635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:43,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521623645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:43,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:43,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:43,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/d505c863518b4c93b5974fffc5c2381a 2024-11-13T18:12:43,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:43,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/1c642e46e4c94a578bf5a97ceba34a3b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/1c642e46e4c94a578bf5a97ceba34a3b 2024-11-13T18:12:43,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:43,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:43,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:43,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/1c642e46e4c94a578bf5a97ceba34a3b, entries=150, sequenceid=102, filesize=30.2 K 2024-11-13T18:12:43,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/d939f2ca07fd49429138b090b2329edf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d939f2ca07fd49429138b090b2329edf 2024-11-13T18:12:43,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d939f2ca07fd49429138b090b2329edf, entries=150, sequenceid=102, filesize=11.7 K 2024-11-13T18:12:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/d505c863518b4c93b5974fffc5c2381a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d505c863518b4c93b5974fffc5c2381a 2024-11-13T18:12:44,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d505c863518b4c93b5974fffc5c2381a, entries=150, sequenceid=102, filesize=11.7 K 2024-11-13T18:12:44,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 634afcda8bfed22b98df3e6e31daa427 in 1163ms, sequenceid=102, compaction requested=true 2024-11-13T18:12:44,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:44,046 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:12:44,046 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:44,060 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:44,061 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/A is initiating minor compaction (all files) 2024-11-13T18:12:44,061 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/A in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:44,061 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/889a7e7d5c574fafa7ba7c2c68ed03ce, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/ee6892e17bbd4a75a3b76ef502e15cfa, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/1c642e46e4c94a578bf5a97ceba34a3b] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=99.3 K 2024-11-13T18:12:44,061 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:44,061 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/889a7e7d5c574fafa7ba7c2c68ed03ce, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/ee6892e17bbd4a75a3b76ef502e15cfa, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/1c642e46e4c94a578bf5a97ceba34a3b] 2024-11-13T18:12:44,063 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:44,064 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/B is initiating minor compaction (all files) 2024-11-13T18:12:44,064 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/B in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:44,064 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/34593026383f4f9c91b340cefd7b9150, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4c4e54e8f9cc4d8aaf5b15b4f5a09e8b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d939f2ca07fd49429138b090b2329edf] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=35.3 K 2024-11-13T18:12:44,065 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 889a7e7d5c574fafa7ba7c2c68ed03ce, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731521559935 2024-11-13T18:12:44,067 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34593026383f4f9c91b340cefd7b9150, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731521559935 2024-11-13T18:12:44,067 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ee6892e17bbd4a75a3b76ef502e15cfa, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731521560493 2024-11-13T18:12:44,070 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c4e54e8f9cc4d8aaf5b15b4f5a09e8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731521560493 2024-11-13T18:12:44,071 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c642e46e4c94a578bf5a97ceba34a3b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731521561698 2024-11-13T18:12:44,071 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d939f2ca07fd49429138b090b2329edf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731521561698 2024-11-13T18:12:44,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:12:44,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:44,113 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:12:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:44,121 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#B#compaction#132 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:44,122 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:44,122 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/494e4eacf37f460e99ea3446fdee3c4f is 50, key is test_row_0/B:col10/1731521561698/Put/seqid=0 2024-11-13T18:12:44,127 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411133ee2f67bf2ce41229c87fa7c4e15faa7_634afcda8bfed22b98df3e6e31daa427 store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:44,130 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411133ee2f67bf2ce41229c87fa7c4e15faa7_634afcda8bfed22b98df3e6e31daa427, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:44,131 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133ee2f67bf2ce41229c87fa7c4e15faa7_634afcda8bfed22b98df3e6e31daa427 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:44,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741978_1154 (size=12207) 2024-11-13T18:12:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:44,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:44,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111321d9b6a30cdb4afab46b24f6a65f7e00_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521562937/Put/seqid=0 2024-11-13T18:12:44,167 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/494e4eacf37f460e99ea3446fdee3c4f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/494e4eacf37f460e99ea3446fdee3c4f 2024-11-13T18:12:44,176 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/B of 634afcda8bfed22b98df3e6e31daa427 into 494e4eacf37f460e99ea3446fdee3c4f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:44,177 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:44,177 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/B, priority=13, startTime=1731521564046; duration=0sec 2024-11-13T18:12:44,177 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:44,177 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:B 2024-11-13T18:12:44,177 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:44,179 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:44,179 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/C is initiating minor compaction (all files) 2024-11-13T18:12:44,179 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/C in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:44,179 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/457aeb6039744b3183060bf4abfa489f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/28d4a095c82c4e02b264f560f57d5ef2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d505c863518b4c93b5974fffc5c2381a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=35.3 K 2024-11-13T18:12:44,180 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 457aeb6039744b3183060bf4abfa489f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731521559935 2024-11-13T18:12:44,180 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28d4a095c82c4e02b264f560f57d5ef2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731521560493 2024-11-13T18:12:44,181 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d505c863518b4c93b5974fffc5c2381a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731521561698 2024-11-13T18:12:44,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741979_1155 (size=4469) 2024-11-13T18:12:44,199 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#A#compaction#133 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:44,200 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/88c659aa60304bf2874f4cb95c7e4f92 is 175, key is test_row_0/A:col10/1731521561698/Put/seqid=0 2024-11-13T18:12:44,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741980_1156 (size=12154) 2024-11-13T18:12:44,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:44,218 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:44,218 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111321d9b6a30cdb4afab46b24f6a65f7e00_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111321d9b6a30cdb4afab46b24f6a65f7e00_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:44,218 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/c008875b8ac94e4bac1828a4393ce9ba is 50, key is test_row_0/C:col10/1731521561698/Put/seqid=0 2024-11-13T18:12:44,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521624215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521624216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741981_1157 (size=31161) 2024-11-13T18:12:44,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/91f41669a1994fd1a4b449bb813c56ce, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:44,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/91f41669a1994fd1a4b449bb813c56ce is 175, key is test_row_0/A:col10/1731521562937/Put/seqid=0 2024-11-13T18:12:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521624221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741982_1158 (size=12207) 2024-11-13T18:12:44,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741983_1159 (size=30955) 2024-11-13T18:12:44,239 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/91f41669a1994fd1a4b449bb813c56ce 2024-11-13T18:12:44,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/2c4e3377fb2c479284dd19936e0a6847 is 50, key is test_row_0/B:col10/1731521562937/Put/seqid=0 2024-11-13T18:12:44,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741984_1160 (size=12001) 2024-11-13T18:12:44,272 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/2c4e3377fb2c479284dd19936e0a6847 2024-11-13T18:12:44,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/12efb45a9be94e0388d4089f5702abf1 is 50, key is test_row_0/C:col10/1731521562937/Put/seqid=0 2024-11-13T18:12:44,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741985_1161 (size=12001) 2024-11-13T18:12:44,295 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/12efb45a9be94e0388d4089f5702abf1 2024-11-13T18:12:44,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/91f41669a1994fd1a4b449bb813c56ce as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/91f41669a1994fd1a4b449bb813c56ce 2024-11-13T18:12:44,308 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/91f41669a1994fd1a4b449bb813c56ce, entries=150, sequenceid=121, filesize=30.2 K 2024-11-13T18:12:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/2c4e3377fb2c479284dd19936e0a6847 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2c4e3377fb2c479284dd19936e0a6847 2024-11-13T18:12:44,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2c4e3377fb2c479284dd19936e0a6847, entries=150, sequenceid=121, filesize=11.7 K 2024-11-13T18:12:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/12efb45a9be94e0388d4089f5702abf1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/12efb45a9be94e0388d4089f5702abf1 2024-11-13T18:12:44,320 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/12efb45a9be94e0388d4089f5702abf1, entries=150, sequenceid=121, filesize=11.7 K 2024-11-13T18:12:44,321 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 634afcda8bfed22b98df3e6e31daa427 in 208ms, sequenceid=121, compaction requested=false 2024-11-13T18:12:44,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:44,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:44,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-13T18:12:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-13T18:12:44,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-13T18:12:44,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.9440 sec 2024-11-13T18:12:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:44,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:12:44,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 4.9550 sec 2024-11-13T18:12:44,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:44,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:44,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:44,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:44,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:44,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:44,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f8bce870584d431ab8eb23799bded1f7_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521564216/Put/seqid=0 2024-11-13T18:12:44,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741986_1162 (size=17284) 2024-11-13T18:12:44,403 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:44,409 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f8bce870584d431ab8eb23799bded1f7_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f8bce870584d431ab8eb23799bded1f7_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:44,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521624400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,410 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/78f570390823416fbcc315e157ca14a9, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:44,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/78f570390823416fbcc315e157ca14a9 is 175, key is test_row_0/A:col10/1731521564216/Put/seqid=0 2024-11-13T18:12:44,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521624408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521624411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741987_1163 (size=48389) 2024-11-13T18:12:44,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521624514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521624524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521624524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,636 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/88c659aa60304bf2874f4cb95c7e4f92 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/88c659aa60304bf2874f4cb95c7e4f92 2024-11-13T18:12:44,645 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/c008875b8ac94e4bac1828a4393ce9ba as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/c008875b8ac94e4bac1828a4393ce9ba 2024-11-13T18:12:44,653 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/A of 634afcda8bfed22b98df3e6e31daa427 into 88c659aa60304bf2874f4cb95c7e4f92(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:44,653 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:44,653 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/A, priority=13, startTime=1731521564046; duration=0sec 2024-11-13T18:12:44,653 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/C of 634afcda8bfed22b98df3e6e31daa427 into c008875b8ac94e4bac1828a4393ce9ba(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:44,653 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:44,653 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/C, priority=13, startTime=1731521564046; duration=0sec 2024-11-13T18:12:44,653 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:44,653 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:A 2024-11-13T18:12:44,653 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:44,653 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:C 2024-11-13T18:12:44,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521624737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521624744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,777 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_1' in 634afcda8bfed22b98df3e6e31daa427 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521624754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,779 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 634afcda8bfed22b98df3e6e31daa427 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521624784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521624784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:44,826 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/78f570390823416fbcc315e157ca14a9 2024-11-13T18:12:44,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/52029bd40ea14733b6f4e6404f21ff97 is 50, key is test_row_0/B:col10/1731521564216/Put/seqid=0 2024-11-13T18:12:44,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741988_1164 (size=12151) 2024-11-13T18:12:44,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/52029bd40ea14733b6f4e6404f21ff97 2024-11-13T18:12:44,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/9be3decddb5a4ca7b9e9a37cdc18e607 is 50, key is test_row_0/C:col10/1731521564216/Put/seqid=0 2024-11-13T18:12:44,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741989_1165 (size=12151) 2024-11-13T18:12:44,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/9be3decddb5a4ca7b9e9a37cdc18e607 2024-11-13T18:12:45,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/78f570390823416fbcc315e157ca14a9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/78f570390823416fbcc315e157ca14a9 2024-11-13T18:12:45,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/78f570390823416fbcc315e157ca14a9, entries=250, sequenceid=137, filesize=47.3 K 2024-11-13T18:12:45,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/52029bd40ea14733b6f4e6404f21ff97 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/52029bd40ea14733b6f4e6404f21ff97 2024-11-13T18:12:45,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/52029bd40ea14733b6f4e6404f21ff97, entries=150, sequenceid=137, filesize=11.9 K 2024-11-13T18:12:45,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/9be3decddb5a4ca7b9e9a37cdc18e607 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/9be3decddb5a4ca7b9e9a37cdc18e607 2024-11-13T18:12:45,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/9be3decddb5a4ca7b9e9a37cdc18e607, entries=150, sequenceid=137, filesize=11.9 K 2024-11-13T18:12:45,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 634afcda8bfed22b98df3e6e31daa427 in 699ms, sequenceid=137, compaction requested=true 2024-11-13T18:12:45,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:45,032 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:45,034 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:45,034 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/A is initiating minor compaction (all files) 2024-11-13T18:12:45,034 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/A in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:45,034 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/88c659aa60304bf2874f4cb95c7e4f92, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/91f41669a1994fd1a4b449bb813c56ce, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/78f570390823416fbcc315e157ca14a9] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=107.9 K 2024-11-13T18:12:45,034 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:45,034 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/88c659aa60304bf2874f4cb95c7e4f92, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/91f41669a1994fd1a4b449bb813c56ce, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/78f570390823416fbcc315e157ca14a9] 2024-11-13T18:12:45,035 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88c659aa60304bf2874f4cb95c7e4f92, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731521561698 2024-11-13T18:12:45,035 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91f41669a1994fd1a4b449bb813c56ce, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731521562937 2024-11-13T18:12:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:45,036 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:45,036 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78f570390823416fbcc315e157ca14a9, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731521564199 2024-11-13T18:12:45,037 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:45,037 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/B is initiating minor compaction (all files) 2024-11-13T18:12:45,037 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/B in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:45,038 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/494e4eacf37f460e99ea3446fdee3c4f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2c4e3377fb2c479284dd19936e0a6847, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/52029bd40ea14733b6f4e6404f21ff97] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=35.5 K 2024-11-13T18:12:45,038 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 494e4eacf37f460e99ea3446fdee3c4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731521561698 2024-11-13T18:12:45,039 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c4e3377fb2c479284dd19936e0a6847, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731521562937 2024-11-13T18:12:45,040 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 52029bd40ea14733b6f4e6404f21ff97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731521564199 2024-11-13T18:12:45,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:45,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:12:45,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:45,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:45,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:45,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:45,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:45,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:45,067 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:45,068 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/8d6beacc789c44c39c2a98cc35bc66b4 is 50, key is test_row_0/B:col10/1731521564216/Put/seqid=0 2024-11-13T18:12:45,071 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:45,140 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111352f3036078ca4f83ace967effbea32f7_634afcda8bfed22b98df3e6e31daa427 store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:45,142 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111352f3036078ca4f83ace967effbea32f7_634afcda8bfed22b98df3e6e31daa427, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:45,143 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111352f3036078ca4f83ace967effbea32f7_634afcda8bfed22b98df3e6e31daa427 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:45,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f88063df52124720bc14fd5ab09088ab_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521565048/Put/seqid=0 2024-11-13T18:12:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521625146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521625149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521625151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741990_1166 (size=12459) 2024-11-13T18:12:45,218 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/8d6beacc789c44c39c2a98cc35bc66b4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8d6beacc789c44c39c2a98cc35bc66b4 2024-11-13T18:12:45,227 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/B of 634afcda8bfed22b98df3e6e31daa427 into 8d6beacc789c44c39c2a98cc35bc66b4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:45,227 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:45,227 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/B, priority=13, startTime=1731521565036; duration=0sec 2024-11-13T18:12:45,227 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:45,227 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:B 2024-11-13T18:12:45,227 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:45,230 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:45,230 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/C is initiating minor compaction (all files) 2024-11-13T18:12:45,230 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/C in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:45,230 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/c008875b8ac94e4bac1828a4393ce9ba, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/12efb45a9be94e0388d4089f5702abf1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/9be3decddb5a4ca7b9e9a37cdc18e607] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=35.5 K 2024-11-13T18:12:45,231 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c008875b8ac94e4bac1828a4393ce9ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731521561698 2024-11-13T18:12:45,231 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 12efb45a9be94e0388d4089f5702abf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731521562937 2024-11-13T18:12:45,231 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be3decddb5a4ca7b9e9a37cdc18e607, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731521564199 2024-11-13T18:12:45,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741991_1167 (size=4469) 2024-11-13T18:12:45,255 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#A#compaction#142 average throughput is 0.13 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:45,255 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/3e5f77c31c654b8bae5b50ca10f3d9fb is 175, key is test_row_0/A:col10/1731521564216/Put/seqid=0 2024-11-13T18:12:45,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521625264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521625264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,269 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521625264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,270 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/e011952f6105410d873434dd0fbebe49 is 50, key is test_row_0/C:col10/1731521564216/Put/seqid=0 2024-11-13T18:12:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741992_1168 (size=14794) 2024-11-13T18:12:45,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:45,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f88063df52124720bc14fd5ab09088ab_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f88063df52124720bc14fd5ab09088ab_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:45,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0215a19cf8324a70b913940bb7c3f922, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:45,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0215a19cf8324a70b913940bb7c3f922 is 175, key is test_row_0/A:col10/1731521565048/Put/seqid=0 2024-11-13T18:12:45,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741993_1169 (size=31413) 2024-11-13T18:12:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741994_1170 (size=12459) 2024-11-13T18:12:45,352 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/e011952f6105410d873434dd0fbebe49 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e011952f6105410d873434dd0fbebe49 2024-11-13T18:12:45,365 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/C of 634afcda8bfed22b98df3e6e31daa427 into e011952f6105410d873434dd0fbebe49(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:45,365 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:45,365 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/C, priority=13, startTime=1731521565036; duration=0sec 2024-11-13T18:12:45,365 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:45,365 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:C 2024-11-13T18:12:45,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741995_1171 (size=39749) 2024-11-13T18:12:45,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0215a19cf8324a70b913940bb7c3f922 2024-11-13T18:12:45,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/b61c63492cb0413793d8f6e414e62468 is 50, key is test_row_0/B:col10/1731521565048/Put/seqid=0 2024-11-13T18:12:45,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741996_1172 (size=12151) 2024-11-13T18:12:45,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/b61c63492cb0413793d8f6e414e62468 2024-11-13T18:12:45,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521625474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521625474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521625474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/dee6760498ab4ac3a50d2e963de91a34 is 50, key is test_row_0/C:col10/1731521565048/Put/seqid=0 2024-11-13T18:12:45,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741997_1173 (size=12151) 2024-11-13T18:12:45,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/dee6760498ab4ac3a50d2e963de91a34 2024-11-13T18:12:45,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0215a19cf8324a70b913940bb7c3f922 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0215a19cf8324a70b913940bb7c3f922 2024-11-13T18:12:45,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0215a19cf8324a70b913940bb7c3f922, entries=200, sequenceid=157, filesize=38.8 K 2024-11-13T18:12:45,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/b61c63492cb0413793d8f6e414e62468 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/b61c63492cb0413793d8f6e414e62468 2024-11-13T18:12:45,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/b61c63492cb0413793d8f6e414e62468, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T18:12:45,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/dee6760498ab4ac3a50d2e963de91a34 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dee6760498ab4ac3a50d2e963de91a34 2024-11-13T18:12:45,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dee6760498ab4ac3a50d2e963de91a34, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T18:12:45,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 634afcda8bfed22b98df3e6e31daa427 in 537ms, sequenceid=157, compaction requested=false 2024-11-13T18:12:45,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:45,768 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/3e5f77c31c654b8bae5b50ca10f3d9fb as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3e5f77c31c654b8bae5b50ca10f3d9fb 2024-11-13T18:12:45,778 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/A of 634afcda8bfed22b98df3e6e31daa427 into 3e5f77c31c654b8bae5b50ca10f3d9fb(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:45,778 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:45,779 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/A, priority=13, startTime=1731521565032; duration=0sec 2024-11-13T18:12:45,779 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:45,779 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:A 2024-11-13T18:12:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:45,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:12:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:45,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e158012ac00a47e1b7646c8b794e4358_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521565149/Put/seqid=0 2024-11-13T18:12:45,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741998_1174 (size=14794) 2024-11-13T18:12:45,851 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:45,856 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e158012ac00a47e1b7646c8b794e4358_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e158012ac00a47e1b7646c8b794e4358_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:45,857 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/a4bb4aed96a44c619fc80092e772a8a7, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:45,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/a4bb4aed96a44c619fc80092e772a8a7 is 175, key is test_row_0/A:col10/1731521565149/Put/seqid=0 2024-11-13T18:12:45,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521625848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521625850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521625862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741999_1175 (size=39749) 2024-11-13T18:12:45,899 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/a4bb4aed96a44c619fc80092e772a8a7 2024-11-13T18:12:45,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/81149015c65847e7a9fd5a3b68a58b9f is 50, key is test_row_0/B:col10/1731521565149/Put/seqid=0 2024-11-13T18:12:45,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742000_1176 (size=12151) 2024-11-13T18:12:45,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521625977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521625977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:45,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521625987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521626189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:46,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521626192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:46,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521626204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:46,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/81149015c65847e7a9fd5a3b68a58b9f 2024-11-13T18:12:46,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/77dfa805bb8a4ddeb2a412cd4c4a72cd is 50, key is test_row_0/C:col10/1731521565149/Put/seqid=0 2024-11-13T18:12:46,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742001_1177 (size=12151) 2024-11-13T18:12:46,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521626504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:46,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521626504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:46,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521626525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:46,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/77dfa805bb8a4ddeb2a412cd4c4a72cd 2024-11-13T18:12:46,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/a4bb4aed96a44c619fc80092e772a8a7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a4bb4aed96a44c619fc80092e772a8a7 2024-11-13T18:12:46,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a4bb4aed96a44c619fc80092e772a8a7, entries=200, sequenceid=175, filesize=38.8 K 2024-11-13T18:12:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/81149015c65847e7a9fd5a3b68a58b9f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/81149015c65847e7a9fd5a3b68a58b9f 2024-11-13T18:12:46,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/81149015c65847e7a9fd5a3b68a58b9f, entries=150, sequenceid=175, filesize=11.9 K 2024-11-13T18:12:46,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/77dfa805bb8a4ddeb2a412cd4c4a72cd as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/77dfa805bb8a4ddeb2a412cd4c4a72cd 2024-11-13T18:12:46,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/77dfa805bb8a4ddeb2a412cd4c4a72cd, entries=150, sequenceid=175, filesize=11.9 K 2024-11-13T18:12:46,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 634afcda8bfed22b98df3e6e31daa427 in 1062ms, sequenceid=175, compaction requested=true 2024-11-13T18:12:46,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:46,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:46,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:46,852 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:46,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:46,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:46,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:46,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:46,852 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:46,855 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:46,855 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/A is initiating minor compaction (all files) 2024-11-13T18:12:46,855 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/A in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:46,855 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3e5f77c31c654b8bae5b50ca10f3d9fb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0215a19cf8324a70b913940bb7c3f922, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a4bb4aed96a44c619fc80092e772a8a7] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=108.3 K 2024-11-13T18:12:46,855 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:46,855 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3e5f77c31c654b8bae5b50ca10f3d9fb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0215a19cf8324a70b913940bb7c3f922, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a4bb4aed96a44c619fc80092e772a8a7] 2024-11-13T18:12:46,855 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:46,856 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/B is initiating minor compaction (all files) 2024-11-13T18:12:46,856 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/B in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:46,856 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8d6beacc789c44c39c2a98cc35bc66b4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/b61c63492cb0413793d8f6e414e62468, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/81149015c65847e7a9fd5a3b68a58b9f] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=35.9 K 2024-11-13T18:12:46,856 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e5f77c31c654b8bae5b50ca10f3d9fb, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731521564199 2024-11-13T18:12:46,856 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d6beacc789c44c39c2a98cc35bc66b4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731521564199 2024-11-13T18:12:46,857 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0215a19cf8324a70b913940bb7c3f922, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731521564396 2024-11-13T18:12:46,858 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b61c63492cb0413793d8f6e414e62468, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731521564396 2024-11-13T18:12:46,859 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4bb4aed96a44c619fc80092e772a8a7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731521565129 2024-11-13T18:12:46,859 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 81149015c65847e7a9fd5a3b68a58b9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731521565129 2024-11-13T18:12:46,869 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:46,871 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411137f91f62f9a2246b0bbc5011e6846bfcc_634afcda8bfed22b98df3e6e31daa427 store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:46,871 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:46,872 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/cd8ee47ce3704714a5b67403785e60f9 is 50, key is test_row_0/B:col10/1731521565149/Put/seqid=0 2024-11-13T18:12:46,874 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411137f91f62f9a2246b0bbc5011e6846bfcc_634afcda8bfed22b98df3e6e31daa427, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:46,874 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137f91f62f9a2246b0bbc5011e6846bfcc_634afcda8bfed22b98df3e6e31daa427 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:46,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742002_1178 (size=4469) 2024-11-13T18:12:46,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742003_1179 (size=12561) 2024-11-13T18:12:46,930 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/cd8ee47ce3704714a5b67403785e60f9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/cd8ee47ce3704714a5b67403785e60f9 2024-11-13T18:12:46,950 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/B of 634afcda8bfed22b98df3e6e31daa427 into cd8ee47ce3704714a5b67403785e60f9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:46,950 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:46,950 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/B, priority=13, startTime=1731521566852; duration=0sec 2024-11-13T18:12:46,950 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:46,950 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:B 2024-11-13T18:12:46,950 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:46,957 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:46,958 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/C is initiating minor compaction (all files) 2024-11-13T18:12:46,958 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/C in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:46,958 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e011952f6105410d873434dd0fbebe49, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dee6760498ab4ac3a50d2e963de91a34, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/77dfa805bb8a4ddeb2a412cd4c4a72cd] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=35.9 K 2024-11-13T18:12:46,965 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e011952f6105410d873434dd0fbebe49, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731521564199 2024-11-13T18:12:46,965 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting dee6760498ab4ac3a50d2e963de91a34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731521564396 2024-11-13T18:12:46,966 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 77dfa805bb8a4ddeb2a412cd4c4a72cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731521565129 2024-11-13T18:12:46,991 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#C#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:46,992 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/8d5a264507614b379402d89c625b2c2b is 50, key is test_row_0/C:col10/1731521565149/Put/seqid=0 2024-11-13T18:12:47,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:12:47,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:47,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:47,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:47,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:47,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:47,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:47,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742004_1180 (size=12561) 2024-11-13T18:12:47,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d94544e4e8244c482a36a70845a232b_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521565859/Put/seqid=0 2024-11-13T18:12:47,077 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/8d5a264507614b379402d89c625b2c2b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/8d5a264507614b379402d89c625b2c2b 2024-11-13T18:12:47,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521627059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521627064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,086 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/C of 634afcda8bfed22b98df3e6e31daa427 into 8d5a264507614b379402d89c625b2c2b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:47,086 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:47,086 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/C, priority=13, startTime=1731521566852; duration=0sec 2024-11-13T18:12:47,086 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:47,086 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:C 2024-11-13T18:12:47,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521627080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742005_1181 (size=14794) 2024-11-13T18:12:47,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521627184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521627184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521627195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,307 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#A#compaction#150 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:47,308 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/5d44869ed8d245b1897e4943b06c1d03 is 175, key is test_row_0/A:col10/1731521565149/Put/seqid=0 2024-11-13T18:12:47,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742006_1182 (size=31515) 2024-11-13T18:12:47,334 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/5d44869ed8d245b1897e4943b06c1d03 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5d44869ed8d245b1897e4943b06c1d03 2024-11-13T18:12:47,348 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/A of 634afcda8bfed22b98df3e6e31daa427 into 5d44869ed8d245b1897e4943b06c1d03(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:47,348 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:47,348 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/A, priority=13, startTime=1731521566851; duration=0sec 2024-11-13T18:12:47,349 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:47,349 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:A 2024-11-13T18:12:47,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521627410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521627410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521627424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,530 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:47,553 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d94544e4e8244c482a36a70845a232b_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d94544e4e8244c482a36a70845a232b_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:47,556 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/cedcc557a97347fbb65bd0253187373d, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:47,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/cedcc557a97347fbb65bd0253187373d is 175, key is test_row_0/A:col10/1731521565859/Put/seqid=0 2024-11-13T18:12:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:12:47,559 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:47,572 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-13T18:12:47,579 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:47,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:47,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:12:47,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742007_1183 (size=39749) 2024-11-13T18:12:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:12:47,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521627724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521627724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:12:47,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:47,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:47,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:47,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:47,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:47,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521627734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:12:47,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:12:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:47,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,018 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/cedcc557a97347fbb65bd0253187373d 2024-11-13T18:12:48,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/8e8b25cba073427bbb16d32d421492bc is 50, key is test_row_0/B:col10/1731521565859/Put/seqid=0 2024-11-13T18:12:48,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:12:48,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742008_1184 (size=12151) 2024-11-13T18:12:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:12:48,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:12:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521628237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:48,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521628241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:48,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521628257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:48,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:12:48,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:48,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/8e8b25cba073427bbb16d32d421492bc 2024-11-13T18:12:48,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:12:48,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:48,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/104b56771dea41088df9e45c0d5df2aa is 50, key is test_row_0/C:col10/1731521565859/Put/seqid=0 2024-11-13T18:12:48,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742009_1185 (size=12151) 2024-11-13T18:12:48,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:12:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:12:48,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521628805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:48,809 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_1' in 634afcda8bfed22b98df3e6e31daa427 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8289 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:48,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521628825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:48,828 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 634afcda8bfed22b98df3e6e31daa427 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8307 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:48,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:48,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:48,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:48,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:12:48,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:48,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:48,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:49,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:12:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:49,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:49,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/104b56771dea41088df9e45c0d5df2aa 2024-11-13T18:12:49,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/cedcc557a97347fbb65bd0253187373d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/cedcc557a97347fbb65bd0253187373d 2024-11-13T18:12:49,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/cedcc557a97347fbb65bd0253187373d, entries=200, sequenceid=194, filesize=38.8 K 2024-11-13T18:12:49,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/8e8b25cba073427bbb16d32d421492bc as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8e8b25cba073427bbb16d32d421492bc 2024-11-13T18:12:49,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8e8b25cba073427bbb16d32d421492bc, entries=150, sequenceid=194, filesize=11.9 K 2024-11-13T18:12:49,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/104b56771dea41088df9e45c0d5df2aa as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/104b56771dea41088df9e45c0d5df2aa 2024-11-13T18:12:49,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/104b56771dea41088df9e45c0d5df2aa, entries=150, sequenceid=194, filesize=11.9 K 2024-11-13T18:12:49,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 634afcda8bfed22b98df3e6e31daa427 in 2032ms, sequenceid=194, compaction requested=false 2024-11-13T18:12:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:49,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:12:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:49,156 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:12:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:49,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:49,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111326f86c46b5b1431ea61f81eb32026d13_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521567047/Put/seqid=0 2024-11-13T18:12:49,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742010_1186 (size=12304) 2024-11-13T18:12:49,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:49,218 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111326f86c46b5b1431ea61f81eb32026d13_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111326f86c46b5b1431ea61f81eb32026d13_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:49,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0b9e93bd72514f09917ea0aac6869976, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:49,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0b9e93bd72514f09917ea0aac6869976 is 175, key is test_row_0/A:col10/1731521567047/Put/seqid=0 2024-11-13T18:12:49,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742011_1187 (size=31105) 2024-11-13T18:12:49,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:49,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521629327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521629329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521629329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521629446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521629454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521629454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,480 INFO [master/fda96cf025dc:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T18:12:49,480 INFO [master/fda96cf025dc:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T18:12:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521629654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,660 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0b9e93bd72514f09917ea0aac6869976 2024-11-13T18:12:49,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521629676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521629668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/adaead4e4c6f433e9592ade4d6e611a9 is 50, key is test_row_0/B:col10/1731521567047/Put/seqid=0 2024-11-13T18:12:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:12:49,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742012_1188 (size=12151) 2024-11-13T18:12:49,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521629965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521629984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521629984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:50,127 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/adaead4e4c6f433e9592ade4d6e611a9 2024-11-13T18:12:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/ff28d351c1ff41019a1380d753cf9753 is 50, key is test_row_0/C:col10/1731521567047/Put/seqid=0 2024-11-13T18:12:50,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742013_1189 (size=12151) 2024-11-13T18:12:50,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521630484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:50,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521630495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:50,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521630495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:50,625 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/ff28d351c1ff41019a1380d753cf9753 2024-11-13T18:12:50,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0b9e93bd72514f09917ea0aac6869976 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0b9e93bd72514f09917ea0aac6869976 2024-11-13T18:12:50,647 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0b9e93bd72514f09917ea0aac6869976, entries=150, sequenceid=211, filesize=30.4 K 2024-11-13T18:12:50,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/adaead4e4c6f433e9592ade4d6e611a9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/adaead4e4c6f433e9592ade4d6e611a9 2024-11-13T18:12:50,657 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/adaead4e4c6f433e9592ade4d6e611a9, entries=150, sequenceid=211, filesize=11.9 K 2024-11-13T18:12:50,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/ff28d351c1ff41019a1380d753cf9753 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/ff28d351c1ff41019a1380d753cf9753 2024-11-13T18:12:50,664 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/ff28d351c1ff41019a1380d753cf9753, entries=150, sequenceid=211, filesize=11.9 K 2024-11-13T18:12:50,667 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 634afcda8bfed22b98df3e6e31daa427 in 1511ms, sequenceid=211, compaction requested=true 2024-11-13T18:12:50,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:50,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:50,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-13T18:12:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-13T18:12:50,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-13T18:12:50,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0880 sec 2024-11-13T18:12:50,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 3.1010 sec 2024-11-13T18:12:51,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:12:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:51,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ac95f2957c8443caaa4bfeba620a04f9_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521569326/Put/seqid=0 2024-11-13T18:12:51,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521631548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521631555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521631556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:51,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742014_1190 (size=12304) 2024-11-13T18:12:51,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521631684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:51,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521631684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:51,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521631684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:12:51,735 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:51,735 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-13T18:12:51,740 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:51,740 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:51,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:12:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:12:51,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:12:51,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:51,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:51,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:51,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:51,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:51,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521631904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:51,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521631914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521631914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:51,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:51,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:52,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:52,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ac95f2957c8443caaa4bfeba620a04f9_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ac95f2957c8443caaa4bfeba620a04f9_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:52,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/622a5d79af2c4f16828a4e841f5df598, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:52,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/622a5d79af2c4f16828a4e841f5df598 is 175, key is test_row_0/A:col10/1731521569326/Put/seqid=0 2024-11-13T18:12:52,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:12:52,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:52,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:52,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:52,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:52,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:12:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:52,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742015_1191 (size=31105) 2024-11-13T18:12:52,066 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/622a5d79af2c4f16828a4e841f5df598 2024-11-13T18:12:52,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/ec1fd97bb8f44dc99f2563b6d68b574a is 50, key is test_row_0/B:col10/1731521569326/Put/seqid=0 2024-11-13T18:12:52,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742016_1192 (size=12151) 2024-11-13T18:12:52,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/ec1fd97bb8f44dc99f2563b6d68b574a 2024-11-13T18:12:52,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/dad1f25c11264ea2848bcb6d4f18b7a2 is 50, key is test_row_0/C:col10/1731521569326/Put/seqid=0 2024-11-13T18:12:52,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742017_1193 (size=12151) 2024-11-13T18:12:52,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/dad1f25c11264ea2848bcb6d4f18b7a2 2024-11-13T18:12:52,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/622a5d79af2c4f16828a4e841f5df598 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/622a5d79af2c4f16828a4e841f5df598 2024-11-13T18:12:52,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/622a5d79af2c4f16828a4e841f5df598, entries=150, sequenceid=230, filesize=30.4 K 2024-11-13T18:12:52,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/ec1fd97bb8f44dc99f2563b6d68b574a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/ec1fd97bb8f44dc99f2563b6d68b574a 2024-11-13T18:12:52,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/ec1fd97bb8f44dc99f2563b6d68b574a, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T18:12:52,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/dad1f25c11264ea2848bcb6d4f18b7a2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dad1f25c11264ea2848bcb6d4f18b7a2 2024-11-13T18:12:52,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dad1f25c11264ea2848bcb6d4f18b7a2, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T18:12:52,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 634afcda8bfed22b98df3e6e31daa427 in 656ms, sequenceid=230, compaction requested=true 2024-11-13T18:12:52,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:52,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:52,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:52,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:52,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:52,165 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:52,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:52,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:12:52,165 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:52,167 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:52,167 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/A is initiating minor compaction (all files) 2024-11-13T18:12:52,167 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:52,167 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/A in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:52,167 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/B is initiating minor compaction (all files) 2024-11-13T18:12:52,167 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/B in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:52,167 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5d44869ed8d245b1897e4943b06c1d03, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/cedcc557a97347fbb65bd0253187373d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0b9e93bd72514f09917ea0aac6869976, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/622a5d79af2c4f16828a4e841f5df598] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=130.3 K 2024-11-13T18:12:52,167 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:52,167 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/cd8ee47ce3704714a5b67403785e60f9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8e8b25cba073427bbb16d32d421492bc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/adaead4e4c6f433e9592ade4d6e611a9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/ec1fd97bb8f44dc99f2563b6d68b574a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=47.9 K 2024-11-13T18:12:52,167 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5d44869ed8d245b1897e4943b06c1d03, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/cedcc557a97347fbb65bd0253187373d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0b9e93bd72514f09917ea0aac6869976, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/622a5d79af2c4f16828a4e841f5df598] 2024-11-13T18:12:52,168 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d44869ed8d245b1897e4943b06c1d03, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731521565129 2024-11-13T18:12:52,168 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd8ee47ce3704714a5b67403785e60f9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731521565129 2024-11-13T18:12:52,168 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting cedcc557a97347fbb65bd0253187373d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731521565843 2024-11-13T18:12:52,168 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e8b25cba073427bbb16d32d421492bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731521565843 2024-11-13T18:12:52,169 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b9e93bd72514f09917ea0aac6869976, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731521567047 2024-11-13T18:12:52,169 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting adaead4e4c6f433e9592ade4d6e611a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731521567047 2024-11-13T18:12:52,169 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 622a5d79af2c4f16828a4e841f5df598, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521569325 2024-11-13T18:12:52,170 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec1fd97bb8f44dc99f2563b6d68b574a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521569325 2024-11-13T18:12:52,185 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:52,196 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#B#compaction#163 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:52,197 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/cf213bfbdfb64bdf91a9b4ebe9a9e597 is 50, key is test_row_0/B:col10/1731521569326/Put/seqid=0 2024-11-13T18:12:52,203 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113c6607472ee9d402c959401da57758a2b_634afcda8bfed22b98df3e6e31daa427 store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:52,206 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113c6607472ee9d402c959401da57758a2b_634afcda8bfed22b98df3e6e31daa427, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:52,206 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c6607472ee9d402c959401da57758a2b_634afcda8bfed22b98df3e6e31daa427 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:52,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:12:52,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:52,221 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:12:52,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:52,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:52,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:52,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:52,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:52,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:52,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:52,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742018_1194 (size=12697) 2024-11-13T18:12:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ce24d5eb84f1429484acde1e37074a1c_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521571554/Put/seqid=0 2024-11-13T18:12:52,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742019_1195 (size=4469) 2024-11-13T18:12:52,272 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#A#compaction#162 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:52,272 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/3f72e055542d406cab4c137f8117aba8 is 175, key is test_row_0/A:col10/1731521569326/Put/seqid=0 2024-11-13T18:12:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742020_1196 (size=12304) 2024-11-13T18:12:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:52,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742021_1197 (size=31651) 2024-11-13T18:12:52,327 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ce24d5eb84f1429484acde1e37074a1c_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ce24d5eb84f1429484acde1e37074a1c_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:52,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/08c888e09f4b40a0ae83db769ee9f578, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/08c888e09f4b40a0ae83db769ee9f578 is 175, key is test_row_0/A:col10/1731521571554/Put/seqid=0 2024-11-13T18:12:52,341 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/3f72e055542d406cab4c137f8117aba8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3f72e055542d406cab4c137f8117aba8 2024-11-13T18:12:52,347 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/A of 634afcda8bfed22b98df3e6e31daa427 into 3f72e055542d406cab4c137f8117aba8(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:52,347 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:52,347 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/A, priority=12, startTime=1731521572165; duration=0sec 2024-11-13T18:12:52,347 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:52,347 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:A 2024-11-13T18:12:52,348 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:52,350 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:52,353 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/C is initiating minor compaction (all files) 2024-11-13T18:12:52,353 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/C in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:52,353 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/8d5a264507614b379402d89c625b2c2b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/104b56771dea41088df9e45c0d5df2aa, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/ff28d351c1ff41019a1380d753cf9753, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dad1f25c11264ea2848bcb6d4f18b7a2] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=47.9 K 2024-11-13T18:12:52,353 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d5a264507614b379402d89c625b2c2b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731521565129 2024-11-13T18:12:52,354 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 104b56771dea41088df9e45c0d5df2aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731521565843 2024-11-13T18:12:52,354 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ff28d351c1ff41019a1380d753cf9753, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731521567047 2024-11-13T18:12:52,354 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting dad1f25c11264ea2848bcb6d4f18b7a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521569325 2024-11-13T18:12:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:12:52,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521632350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:52,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521632351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521632352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:52,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742022_1198 (size=31105) 2024-11-13T18:12:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:52,371 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:52,372 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/07c8dacec2294df189a3342dec3ead6e is 50, key is test_row_0/C:col10/1731521569326/Put/seqid=0 2024-11-13T18:12:52,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742023_1199 (size=12697) 2024-11-13T18:12:52,439 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/07c8dacec2294df189a3342dec3ead6e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/07c8dacec2294df189a3342dec3ead6e 2024-11-13T18:12:52,447 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/C of 634afcda8bfed22b98df3e6e31daa427 into 07c8dacec2294df189a3342dec3ead6e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:52,447 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:52,447 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/C, priority=12, startTime=1731521572165; duration=0sec 2024-11-13T18:12:52,447 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:52,447 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:C 2024-11-13T18:12:52,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521632474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521632474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521632474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:52,665 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/cf213bfbdfb64bdf91a9b4ebe9a9e597 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/cf213bfbdfb64bdf91a9b4ebe9a9e597 2024-11-13T18:12:52,677 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/B of 634afcda8bfed22b98df3e6e31daa427 into cf213bfbdfb64bdf91a9b4ebe9a9e597(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:52,677 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:52,677 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/B, priority=12, startTime=1731521572165; duration=0sec 2024-11-13T18:12:52,678 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:52,678 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:B 2024-11-13T18:12:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521632684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521632684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:52,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521632684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:52,772 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/08c888e09f4b40a0ae83db769ee9f578 2024-11-13T18:12:52,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/96cb19197afc472584a907bfa3a40b5c is 50, key is test_row_0/B:col10/1731521571554/Put/seqid=0 2024-11-13T18:12:52,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742024_1200 (size=12151) 2024-11-13T18:12:52,858 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/96cb19197afc472584a907bfa3a40b5c 2024-11-13T18:12:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:12:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/4d751164b588441da677cc3012620266 is 50, key is test_row_0/C:col10/1731521571554/Put/seqid=0 2024-11-13T18:12:52,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742025_1201 (size=12151) 2024-11-13T18:12:52,933 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/4d751164b588441da677cc3012620266 2024-11-13T18:12:52,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/08c888e09f4b40a0ae83db769ee9f578 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/08c888e09f4b40a0ae83db769ee9f578 2024-11-13T18:12:52,948 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/08c888e09f4b40a0ae83db769ee9f578, entries=150, sequenceid=243, filesize=30.4 K 2024-11-13T18:12:52,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/96cb19197afc472584a907bfa3a40b5c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/96cb19197afc472584a907bfa3a40b5c 2024-11-13T18:12:52,955 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/96cb19197afc472584a907bfa3a40b5c, entries=150, sequenceid=243, filesize=11.9 K 2024-11-13T18:12:52,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/4d751164b588441da677cc3012620266 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/4d751164b588441da677cc3012620266 2024-11-13T18:12:52,965 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/4d751164b588441da677cc3012620266, entries=150, sequenceid=243, filesize=11.9 K 2024-11-13T18:12:52,966 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 634afcda8bfed22b98df3e6e31daa427 in 745ms, sequenceid=243, compaction requested=false 2024-11-13T18:12:52,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:52,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:52,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-13T18:12:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-13T18:12:52,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-13T18:12:52,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2330 sec 2024-11-13T18:12:52,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.2420 sec 2024-11-13T18:12:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:53,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:12:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:53,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111382decf0dfcc548b9b9ce31f2cc28621b_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521572319/Put/seqid=0 2024-11-13T18:12:53,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521633052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742026_1202 (size=14994) 2024-11-13T18:12:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,081 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:53,086 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111382decf0dfcc548b9b9ce31f2cc28621b_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111382decf0dfcc548b9b9ce31f2cc28621b_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:53,087 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/221c854b95b34ee7a36aa383530d3ad7, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:53,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/221c854b95b34ee7a36aa383530d3ad7 is 175, key is test_row_0/A:col10/1731521572319/Put/seqid=0 2024-11-13T18:12:53,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521633077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521633080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742027_1203 (size=39949) 2024-11-13T18:12:53,130 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/221c854b95b34ee7a36aa383530d3ad7 2024-11-13T18:12:53,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/efc3cc5b9496475fa8ae1cbc40a28a4f is 50, key is test_row_0/B:col10/1731521572319/Put/seqid=0 2024-11-13T18:12:53,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521633184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742028_1204 (size=12301) 2024-11-13T18:12:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521633195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/efc3cc5b9496475fa8ae1cbc40a28a4f 2024-11-13T18:12:53,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521633204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/d2a2850a86bd4653bd6770842ae6459f is 50, key is test_row_0/C:col10/1731521572319/Put/seqid=0 2024-11-13T18:12:53,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742029_1205 (size=12301) 2024-11-13T18:12:53,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521633394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521633414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521633424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/d2a2850a86bd4653bd6770842ae6459f 2024-11-13T18:12:53,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/221c854b95b34ee7a36aa383530d3ad7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/221c854b95b34ee7a36aa383530d3ad7 2024-11-13T18:12:53,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521633716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/221c854b95b34ee7a36aa383530d3ad7, entries=200, sequenceid=266, filesize=39.0 K 2024-11-13T18:12:53,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/efc3cc5b9496475fa8ae1cbc40a28a4f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/efc3cc5b9496475fa8ae1cbc40a28a4f 2024-11-13T18:12:53,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521633734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521633734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:53,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/efc3cc5b9496475fa8ae1cbc40a28a4f, entries=150, sequenceid=266, filesize=12.0 K 2024-11-13T18:12:53,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/d2a2850a86bd4653bd6770842ae6459f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d2a2850a86bd4653bd6770842ae6459f 2024-11-13T18:12:53,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d2a2850a86bd4653bd6770842ae6459f, entries=150, sequenceid=266, filesize=12.0 K 2024-11-13T18:12:53,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 634afcda8bfed22b98df3e6e31daa427 in 757ms, sequenceid=266, compaction requested=true 2024-11-13T18:12:53,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:53,766 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:53,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:53,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:53,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:53,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:53,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:53,767 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:53,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:53,769 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:53,769 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/A is initiating minor compaction (all files) 2024-11-13T18:12:53,769 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/A in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:53,769 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3f72e055542d406cab4c137f8117aba8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/08c888e09f4b40a0ae83db769ee9f578, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/221c854b95b34ee7a36aa383530d3ad7] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=100.3 K 2024-11-13T18:12:53,769 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:53,769 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3f72e055542d406cab4c137f8117aba8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/08c888e09f4b40a0ae83db769ee9f578, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/221c854b95b34ee7a36aa383530d3ad7] 2024-11-13T18:12:53,769 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:53,769 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f72e055542d406cab4c137f8117aba8, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521569325 2024-11-13T18:12:53,769 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/B is initiating minor compaction (all files) 2024-11-13T18:12:53,769 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/B in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:53,770 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/cf213bfbdfb64bdf91a9b4ebe9a9e597, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/96cb19197afc472584a907bfa3a40b5c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/efc3cc5b9496475fa8ae1cbc40a28a4f] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=36.3 K 2024-11-13T18:12:53,770 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08c888e09f4b40a0ae83db769ee9f578, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731521571546 2024-11-13T18:12:53,770 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting cf213bfbdfb64bdf91a9b4ebe9a9e597, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521569325 2024-11-13T18:12:53,770 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221c854b95b34ee7a36aa383530d3ad7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731521572319 2024-11-13T18:12:53,770 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 96cb19197afc472584a907bfa3a40b5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731521571546 2024-11-13T18:12:53,771 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting efc3cc5b9496475fa8ae1cbc40a28a4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731521572319 2024-11-13T18:12:53,782 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:53,801 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:53,802 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/123a7cf1fa59430ea7b0eede2950ba32 is 50, key is test_row_0/B:col10/1731521572319/Put/seqid=0 2024-11-13T18:12:53,813 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411136469a414d05c4408b4dcf6f0b63842ec_634afcda8bfed22b98df3e6e31daa427 store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:53,816 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411136469a414d05c4408b4dcf6f0b63842ec_634afcda8bfed22b98df3e6e31daa427, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:53,816 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136469a414d05c4408b4dcf6f0b63842ec_634afcda8bfed22b98df3e6e31daa427 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:53,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742030_1206 (size=12949) 2024-11-13T18:12:53,855 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/123a7cf1fa59430ea7b0eede2950ba32 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/123a7cf1fa59430ea7b0eede2950ba32 2024-11-13T18:12:53,864 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/B of 634afcda8bfed22b98df3e6e31daa427 into 123a7cf1fa59430ea7b0eede2950ba32(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:53,864 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:53,864 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/B, priority=13, startTime=1731521573767; duration=0sec 2024-11-13T18:12:53,865 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:53,865 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:B 2024-11-13T18:12:53,865 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:53,868 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:53,868 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/C is initiating minor compaction (all files) 2024-11-13T18:12:53,868 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/C in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:53,868 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/07c8dacec2294df189a3342dec3ead6e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/4d751164b588441da677cc3012620266, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d2a2850a86bd4653bd6770842ae6459f] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=36.3 K 2024-11-13T18:12:53,868 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c8dacec2294df189a3342dec3ead6e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731521569325 2024-11-13T18:12:53,869 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d751164b588441da677cc3012620266, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731521571546 2024-11-13T18:12:53,869 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d2a2850a86bd4653bd6770842ae6459f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731521572319 2024-11-13T18:12:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:12:53,885 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:53,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-13T18:12:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:12:53,894 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:53,894 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:53,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:53,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742031_1207 (size=4469) 2024-11-13T18:12:53,912 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#A#compaction#171 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:53,912 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#C#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:53,913 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/c5f932b5804e447a972ecfa1a3977dda is 175, key is test_row_0/A:col10/1731521572319/Put/seqid=0 2024-11-13T18:12:53,913 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/fe435e12d24c41e5a589e9455380a733 is 50, key is test_row_0/C:col10/1731521572319/Put/seqid=0 2024-11-13T18:12:53,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742032_1208 (size=12949) 2024-11-13T18:12:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:12:53,996 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/fe435e12d24c41e5a589e9455380a733 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fe435e12d24c41e5a589e9455380a733 2024-11-13T18:12:53,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742033_1209 (size=31903) 2024-11-13T18:12:54,004 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/C of 634afcda8bfed22b98df3e6e31daa427 into fe435e12d24c41e5a589e9455380a733(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:54,004 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:54,004 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/C, priority=13, startTime=1731521573767; duration=0sec 2024-11-13T18:12:54,005 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:54,005 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:C 2024-11-13T18:12:54,009 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/c5f932b5804e447a972ecfa1a3977dda as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5f932b5804e447a972ecfa1a3977dda 2024-11-13T18:12:54,016 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/A of 634afcda8bfed22b98df3e6e31daa427 into c5f932b5804e447a972ecfa1a3977dda(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:54,016 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:54,016 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/A, priority=13, startTime=1731521573766; duration=0sec 2024-11-13T18:12:54,016 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:54,016 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:A 2024-11-13T18:12:54,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T18:12:54,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:54,053 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:12:54,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:54,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:54,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:54,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:54,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:54,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:54,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e701dd56b669472aa9f709cb5bbb3827_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521573073/Put/seqid=0 2024-11-13T18:12:54,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742034_1210 (size=12454) 2024-11-13T18:12:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:12:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:54,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:54,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521634281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521634283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521634288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521634394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521634394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521634404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:12:54,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:54,536 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e701dd56b669472aa9f709cb5bbb3827_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e701dd56b669472aa9f709cb5bbb3827_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:54,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/47f87cb160514b1e9fb9db1aaf3d27c5, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:54,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/47f87cb160514b1e9fb9db1aaf3d27c5 is 175, key is test_row_0/A:col10/1731521573073/Put/seqid=0 2024-11-13T18:12:54,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742035_1211 (size=31255) 2024-11-13T18:12:54,570 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/47f87cb160514b1e9fb9db1aaf3d27c5 2024-11-13T18:12:54,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/5dda75549e6e4fda91c0b986695dc54b is 50, key is test_row_0/B:col10/1731521573073/Put/seqid=0 2024-11-13T18:12:54,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742036_1212 (size=12301) 2024-11-13T18:12:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521634604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521634604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521634615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521634914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521634914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:54,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521634925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:55,005 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/5dda75549e6e4fda91c0b986695dc54b 2024-11-13T18:12:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:12:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/20e0def1ba144b74a61937589d436549 is 50, key is test_row_0/C:col10/1731521573073/Put/seqid=0 2024-11-13T18:12:55,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742037_1213 (size=12301) 2024-11-13T18:12:55,112 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/20e0def1ba144b74a61937589d436549 2024-11-13T18:12:55,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/47f87cb160514b1e9fb9db1aaf3d27c5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/47f87cb160514b1e9fb9db1aaf3d27c5 2024-11-13T18:12:55,181 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/47f87cb160514b1e9fb9db1aaf3d27c5, entries=150, sequenceid=282, filesize=30.5 K 2024-11-13T18:12:55,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/5dda75549e6e4fda91c0b986695dc54b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5dda75549e6e4fda91c0b986695dc54b 2024-11-13T18:12:55,241 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5dda75549e6e4fda91c0b986695dc54b, entries=150, sequenceid=282, filesize=12.0 K 2024-11-13T18:12:55,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/20e0def1ba144b74a61937589d436549 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/20e0def1ba144b74a61937589d436549 2024-11-13T18:12:55,255 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/20e0def1ba144b74a61937589d436549, entries=150, sequenceid=282, filesize=12.0 K 2024-11-13T18:12:55,265 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 634afcda8bfed22b98df3e6e31daa427 in 1212ms, sequenceid=282, compaction requested=false 2024-11-13T18:12:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-13T18:12:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-13T18:12:55,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-13T18:12:55,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3820 sec 2024-11-13T18:12:55,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.4050 sec 2024-11-13T18:12:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:55,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:12:55,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:55,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:55,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:55,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:55,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:55,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:55,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132c6fc83d3fca44efa34c11180e7c92f7_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521574281/Put/seqid=0 2024-11-13T18:12:55,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521635459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:55,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521635465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:55,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521635471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:55,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742038_1214 (size=14994) 2024-11-13T18:12:55,505 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:55,511 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132c6fc83d3fca44efa34c11180e7c92f7_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c6fc83d3fca44efa34c11180e7c92f7_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:55,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/f4eef13adebe4fe195856b9f307b2cc6, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:55,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/f4eef13adebe4fe195856b9f307b2cc6 is 175, key is test_row_0/A:col10/1731521574281/Put/seqid=0 2024-11-13T18:12:55,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742039_1215 (size=39949) 2024-11-13T18:12:55,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521635574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:55,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521635584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:55,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521635584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521635784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:55,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521635794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:55,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521635796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:55,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/f4eef13adebe4fe195856b9f307b2cc6 2024-11-13T18:12:55,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/665b6f0551a644e5ae637e71b2a0518e is 50, key is test_row_0/B:col10/1731521574281/Put/seqid=0 2024-11-13T18:12:56,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742040_1216 (size=12301) 2024-11-13T18:12:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:12:56,045 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:56,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-13T18:12:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:12:56,048 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:56,049 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:56,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521636104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:56,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521636104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521636104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:12:56,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:12:56,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:56,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:56,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:56,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:56,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:12:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:56,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:56,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:12:56,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/665b6f0551a644e5ae637e71b2a0518e 2024-11-13T18:12:56,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/147f9f3d1ec34a3cbf5bbf419fd708c6 is 50, key is test_row_0/C:col10/1731521574281/Put/seqid=0 2024-11-13T18:12:56,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742041_1217 (size=12301) 2024-11-13T18:12:56,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/147f9f3d1ec34a3cbf5bbf419fd708c6 2024-11-13T18:12:56,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/f4eef13adebe4fe195856b9f307b2cc6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/f4eef13adebe4fe195856b9f307b2cc6 2024-11-13T18:12:56,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/f4eef13adebe4fe195856b9f307b2cc6, entries=200, sequenceid=302, filesize=39.0 K 2024-11-13T18:12:56,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/665b6f0551a644e5ae637e71b2a0518e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/665b6f0551a644e5ae637e71b2a0518e 2024-11-13T18:12:56,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/665b6f0551a644e5ae637e71b2a0518e, entries=150, sequenceid=302, filesize=12.0 K 2024-11-13T18:12:56,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/147f9f3d1ec34a3cbf5bbf419fd708c6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/147f9f3d1ec34a3cbf5bbf419fd708c6 2024-11-13T18:12:56,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/147f9f3d1ec34a3cbf5bbf419fd708c6, entries=150, sequenceid=302, filesize=12.0 K 2024-11-13T18:12:56,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 634afcda8bfed22b98df3e6e31daa427 in 1052ms, sequenceid=302, compaction requested=true 2024-11-13T18:12:56,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:56,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:56,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:56,484 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:56,484 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:56,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:56,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:56,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:56,485 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:56,486 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/A is initiating minor compaction (all files) 2024-11-13T18:12:56,486 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/A in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:56,486 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5f932b5804e447a972ecfa1a3977dda, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/47f87cb160514b1e9fb9db1aaf3d27c5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/f4eef13adebe4fe195856b9f307b2cc6] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=100.7 K 2024-11-13T18:12:56,486 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:56,486 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5f932b5804e447a972ecfa1a3977dda, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/47f87cb160514b1e9fb9db1aaf3d27c5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/f4eef13adebe4fe195856b9f307b2cc6] 2024-11-13T18:12:56,487 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:56,487 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5f932b5804e447a972ecfa1a3977dda, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731521572319 2024-11-13T18:12:56,487 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/B is initiating minor compaction (all files) 2024-11-13T18:12:56,487 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/B in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:56,487 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/123a7cf1fa59430ea7b0eede2950ba32, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5dda75549e6e4fda91c0b986695dc54b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/665b6f0551a644e5ae637e71b2a0518e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=36.7 K 2024-11-13T18:12:56,487 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47f87cb160514b1e9fb9db1aaf3d27c5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731521573050 2024-11-13T18:12:56,488 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4eef13adebe4fe195856b9f307b2cc6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731521574275 2024-11-13T18:12:56,488 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 123a7cf1fa59430ea7b0eede2950ba32, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731521572319 2024-11-13T18:12:56,488 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dda75549e6e4fda91c0b986695dc54b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731521573050 2024-11-13T18:12:56,489 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 665b6f0551a644e5ae637e71b2a0518e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731521574275 2024-11-13T18:12:56,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:56,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:12:56,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:56,512 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:12:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:56,515 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:56,517 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#B#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:56,521 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/8a809781913f498b916c8e10771d9a10 is 50, key is test_row_0/B:col10/1731521574281/Put/seqid=0 2024-11-13T18:12:56,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138e0bb57474d84beea3f346d2a449a77e_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521575433/Put/seqid=0 2024-11-13T18:12:56,536 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111385df0a16b87b4be8ab6196c197885c6a_634afcda8bfed22b98df3e6e31daa427 store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:56,547 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111385df0a16b87b4be8ab6196c197885c6a_634afcda8bfed22b98df3e6e31daa427, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:56,547 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111385df0a16b87b4be8ab6196c197885c6a_634afcda8bfed22b98df3e6e31daa427 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:56,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742042_1218 (size=13051) 2024-11-13T18:12:56,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742043_1219 (size=12454) 2024-11-13T18:12:56,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742044_1220 (size=4469) 2024-11-13T18:12:56,608 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#A#compaction#180 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:56,609 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/6f1e02a63bf74ee79e41168595f24d31 is 175, key is test_row_0/A:col10/1731521574281/Put/seqid=0 2024-11-13T18:12:56,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:56,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742045_1221 (size=32005) 2024-11-13T18:12:56,665 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/6f1e02a63bf74ee79e41168595f24d31 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6f1e02a63bf74ee79e41168595f24d31 2024-11-13T18:12:56,672 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/A of 634afcda8bfed22b98df3e6e31daa427 into 6f1e02a63bf74ee79e41168595f24d31(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:56,672 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:56,672 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/A, priority=13, startTime=1731521576484; duration=0sec 2024-11-13T18:12:56,672 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:56,672 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:A 2024-11-13T18:12:56,672 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:12:56,676 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:56,677 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/C is initiating minor compaction (all files) 2024-11-13T18:12:56,677 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/C in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:56,677 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fe435e12d24c41e5a589e9455380a733, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/20e0def1ba144b74a61937589d436549, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/147f9f3d1ec34a3cbf5bbf419fd708c6] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=36.7 K 2024-11-13T18:12:56,677 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe435e12d24c41e5a589e9455380a733, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731521572319 2024-11-13T18:12:56,680 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20e0def1ba144b74a61937589d436549, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731521573050 2024-11-13T18:12:56,681 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 147f9f3d1ec34a3cbf5bbf419fd708c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731521574275 2024-11-13T18:12:56,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521636710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:56,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521636712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:56,726 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:56,726 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/940de46e5d6a47d5bfd434cc23da26c7 is 50, key is test_row_0/C:col10/1731521574281/Put/seqid=0 2024-11-13T18:12:56,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521636721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:56,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:56,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742046_1222 (size=13051) 2024-11-13T18:12:56,786 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/940de46e5d6a47d5bfd434cc23da26c7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/940de46e5d6a47d5bfd434cc23da26c7 2024-11-13T18:12:56,795 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/C of 634afcda8bfed22b98df3e6e31daa427 into 940de46e5d6a47d5bfd434cc23da26c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:56,795 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:56,795 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/C, priority=13, startTime=1731521576485; duration=0sec 2024-11-13T18:12:56,795 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:56,795 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:C 2024-11-13T18:12:56,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521636824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:56,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521636834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:56,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521636845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:56,965 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/8a809781913f498b916c8e10771d9a10 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8a809781913f498b916c8e10771d9a10 2024-11-13T18:12:56,975 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/B of 634afcda8bfed22b98df3e6e31daa427 into 8a809781913f498b916c8e10771d9a10(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:56,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:56,975 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/B, priority=13, startTime=1731521576484; duration=0sec 2024-11-13T18:12:56,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:56,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:B 2024-11-13T18:12:56,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:57,006 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138e0bb57474d84beea3f346d2a449a77e_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138e0bb57474d84beea3f346d2a449a77e_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:57,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/c5fc7fd50c40442d9100a0ba016c3e2c, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:57,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/c5fc7fd50c40442d9100a0ba016c3e2c is 175, key is test_row_0/A:col10/1731521575433/Put/seqid=0 2024-11-13T18:12:57,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742047_1223 (size=31255) 2024-11-13T18:12:57,031 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/c5fc7fd50c40442d9100a0ba016c3e2c 2024-11-13T18:12:57,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521637034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521637044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/1e0c4b59b16e45c2b62a98e49fda8d10 is 50, key is test_row_0/B:col10/1731521575433/Put/seqid=0 2024-11-13T18:12:57,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521637054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742048_1224 (size=12301) 2024-11-13T18:12:57,068 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/1e0c4b59b16e45c2b62a98e49fda8d10 2024-11-13T18:12:57,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/5522c174ca8148a8963ce5d194f91b43 is 50, key is test_row_0/C:col10/1731521575433/Put/seqid=0 2024-11-13T18:12:57,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742049_1225 (size=12301) 2024-11-13T18:12:57,122 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/5522c174ca8148a8963ce5d194f91b43 2024-11-13T18:12:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/c5fc7fd50c40442d9100a0ba016c3e2c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5fc7fd50c40442d9100a0ba016c3e2c 2024-11-13T18:12:57,140 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5fc7fd50c40442d9100a0ba016c3e2c, entries=150, sequenceid=314, filesize=30.5 K 2024-11-13T18:12:57,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/1e0c4b59b16e45c2b62a98e49fda8d10 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/1e0c4b59b16e45c2b62a98e49fda8d10 2024-11-13T18:12:57,155 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/1e0c4b59b16e45c2b62a98e49fda8d10, entries=150, sequenceid=314, filesize=12.0 K 2024-11-13T18:12:57,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/5522c174ca8148a8963ce5d194f91b43 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5522c174ca8148a8963ce5d194f91b43 2024-11-13T18:12:57,172 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5522c174ca8148a8963ce5d194f91b43, entries=150, sequenceid=314, filesize=12.0 K 2024-11-13T18:12:57,174 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 634afcda8bfed22b98df3e6e31daa427 in 662ms, sequenceid=314, compaction requested=false 2024-11-13T18:12:57,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:57,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:57,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-13T18:12:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-13T18:12:57,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-13T18:12:57,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1290 sec 2024-11-13T18:12:57,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.1360 sec 2024-11-13T18:12:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:12:57,192 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:57,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-13T18:12:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:12:57,198 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:57,199 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:57,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:12:57,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521576696, max timestamp=1731521577351Num uniques 859; 2024-11-13T18:12:57,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:12:57,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521576696, max timestamp=1731521577351Num uniques 859; 2024-11-13T18:12:57,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T18:12:57,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:12:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:57,356 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=51.44 KB heapSize=144.02 KB 2024-11-13T18:12:57,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:57,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:57,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:57,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:57,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:57,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:57,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b3ae6a2a42d6412c9239975617abbded_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521576718/Put/seqid=0 2024-11-13T18:12:57,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742050_1226 (size=14994) 2024-11-13T18:12:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:57,409 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b3ae6a2a42d6412c9239975617abbded_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b3ae6a2a42d6412c9239975617abbded_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:57,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/6d1579bebc7749b7a715a2e7fa1a8d0a, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:57,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/6d1579bebc7749b7a715a2e7fa1a8d0a is 175, key is test_row_0/A:col10/1731521576718/Put/seqid=0 2024-11-13T18:12:57,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742051_1227 (size=39949) 2024-11-13T18:12:57,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521577397, max timestamp=1731521577429Num uniques 859; 2024-11-13T18:12:57,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:12:57,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521577397, max timestamp=1731521577429Num uniques 859; 2024-11-13T18:12:57,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:12:57,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521577397, max timestamp=1731521577429Num uniques 859; 2024-11-13T18:12:57,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:12:57,444 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/6d1579bebc7749b7a715a2e7fa1a8d0a 2024-11-13T18:12:57,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/488b1680085e4d1c97a181b4b35e7587 is 50, key is test_row_0/B:col10/1731521576718/Put/seqid=0 2024-11-13T18:12:57,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742052_1228 (size=12301) 2024-11-13T18:12:57,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521637488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,501 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/488b1680085e4d1c97a181b4b35e7587 2024-11-13T18:12:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521637499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521637501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:12:57,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/7161444f99024ea6bd4e8a2c5610e27f is 50, key is test_row_0/C:col10/1731521576718/Put/seqid=0 2024-11-13T18:12:57,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742053_1229 (size=12301) 2024-11-13T18:12:57,574 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/7161444f99024ea6bd4e8a2c5610e27f 2024-11-13T18:12:57,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/6d1579bebc7749b7a715a2e7fa1a8d0a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6d1579bebc7749b7a715a2e7fa1a8d0a 2024-11-13T18:12:57,592 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6d1579bebc7749b7a715a2e7fa1a8d0a, entries=200, sequenceid=338, filesize=39.0 K 2024-11-13T18:12:57,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/488b1680085e4d1c97a181b4b35e7587 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/488b1680085e4d1c97a181b4b35e7587 2024-11-13T18:12:57,598 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/488b1680085e4d1c97a181b4b35e7587, entries=150, sequenceid=338, filesize=12.0 K 2024-11-13T18:12:57,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/7161444f99024ea6bd4e8a2c5610e27f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/7161444f99024ea6bd4e8a2c5610e27f 2024-11-13T18:12:57,605 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/7161444f99024ea6bd4e8a2c5610e27f, entries=150, sequenceid=338, filesize=12.0 K 2024-11-13T18:12:57,606 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 634afcda8bfed22b98df3e6e31daa427 in 249ms, sequenceid=338, compaction requested=true 2024-11-13T18:12:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-13T18:12:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-13T18:12:57,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-13T18:12:57,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 408 msec 2024-11-13T18:12:57,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 417 msec 2024-11-13T18:12:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:57,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T18:12:57,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:57,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:57,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:57,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:57,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:57,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:57,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113965ac7f1bc854a028d3f91af34cfdbf6_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521577495/Put/seqid=0 2024-11-13T18:12:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521637635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521637643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742054_1230 (size=14994) 2024-11-13T18:12:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521637645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,668 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113965ac7f1bc854a028d3f91af34cfdbf6_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113965ac7f1bc854a028d3f91af34cfdbf6_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:57,669 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:57,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0 is 175, key is test_row_0/A:col10/1731521577495/Put/seqid=0 2024-11-13T18:12:57,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742055_1231 (size=39949) 2024-11-13T18:12:57,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0 2024-11-13T18:12:57,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/f9e917dfc52849f08a278e13fbadc5d8 is 50, key is test_row_0/B:col10/1731521577495/Put/seqid=0 2024-11-13T18:12:57,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521637754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742056_1232 (size=12301) 2024-11-13T18:12:57,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/f9e917dfc52849f08a278e13fbadc5d8 2024-11-13T18:12:57,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521637764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521637764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:57,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/41a78edbe609450da18e1bc4f4d94916 is 50, key is test_row_0/C:col10/1731521577495/Put/seqid=0 2024-11-13T18:12:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:12:57,824 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:12:57,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:12:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-13T18:12:57,827 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:12:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:12:57,828 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:12:57,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:12:57,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742057_1233 (size=12301) 2024-11-13T18:12:57,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/41a78edbe609450da18e1bc4f4d94916 2024-11-13T18:12:57,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0 2024-11-13T18:12:57,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0, entries=200, sequenceid=374, filesize=39.0 K 2024-11-13T18:12:57,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/f9e917dfc52849f08a278e13fbadc5d8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/f9e917dfc52849f08a278e13fbadc5d8 2024-11-13T18:12:57,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/f9e917dfc52849f08a278e13fbadc5d8, entries=150, sequenceid=374, filesize=12.0 K 2024-11-13T18:12:57,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/41a78edbe609450da18e1bc4f4d94916 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/41a78edbe609450da18e1bc4f4d94916 2024-11-13T18:12:57,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/41a78edbe609450da18e1bc4f4d94916, entries=150, sequenceid=374, filesize=12.0 K 2024-11-13T18:12:57,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 634afcda8bfed22b98df3e6e31daa427 in 271ms, sequenceid=374, compaction requested=true 2024-11-13T18:12:57,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:57,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:57,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:57,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:57,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:57,883 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:57,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:57,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:57,883 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:57,885 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:57,885 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/A is initiating minor compaction (all files) 2024-11-13T18:12:57,885 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/A in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:57,885 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6f1e02a63bf74ee79e41168595f24d31, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5fc7fd50c40442d9100a0ba016c3e2c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6d1579bebc7749b7a715a2e7fa1a8d0a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=139.8 K 2024-11-13T18:12:57,885 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:57,885 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6f1e02a63bf74ee79e41168595f24d31, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5fc7fd50c40442d9100a0ba016c3e2c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6d1579bebc7749b7a715a2e7fa1a8d0a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0] 2024-11-13T18:12:57,886 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:57,886 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f1e02a63bf74ee79e41168595f24d31, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731521574275 2024-11-13T18:12:57,886 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/B is initiating minor compaction (all files) 2024-11-13T18:12:57,886 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/B in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:57,887 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8a809781913f498b916c8e10771d9a10, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/1e0c4b59b16e45c2b62a98e49fda8d10, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/488b1680085e4d1c97a181b4b35e7587, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/f9e917dfc52849f08a278e13fbadc5d8] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=48.8 K 2024-11-13T18:12:57,887 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5fc7fd50c40442d9100a0ba016c3e2c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731521575433 2024-11-13T18:12:57,888 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a809781913f498b916c8e10771d9a10, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731521574275 2024-11-13T18:12:57,888 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d1579bebc7749b7a715a2e7fa1a8d0a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731521576696 2024-11-13T18:12:57,889 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0c4b59b16e45c2b62a98e49fda8d10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731521575433 2024-11-13T18:12:57,889 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 488b1680085e4d1c97a181b4b35e7587, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731521576708 2024-11-13T18:12:57,890 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e1e7f3b71d7492bbfac24f5bb6bb7c0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521577462 2024-11-13T18:12:57,891 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e917dfc52849f08a278e13fbadc5d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521577483 2024-11-13T18:12:57,912 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:57,915 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:57,915 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/d563a427bee24d2b8f697baa3b5774fe is 50, key is test_row_0/B:col10/1731521577495/Put/seqid=0 2024-11-13T18:12:57,929 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113a0d182f76066407ea5200ac1ea92ac45_634afcda8bfed22b98df3e6e31daa427 store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:57,931 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113a0d182f76066407ea5200ac1ea92ac45_634afcda8bfed22b98df3e6e31daa427, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:57,932 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a0d182f76066407ea5200ac1ea92ac45_634afcda8bfed22b98df3e6e31daa427 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:12:57,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742058_1234 (size=13187) 2024-11-13T18:12:57,969 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/d563a427bee24d2b8f697baa3b5774fe as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d563a427bee24d2b8f697baa3b5774fe 2024-11-13T18:12:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:57,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:12:57,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:57,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:57,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:57,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:57,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:57,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:12:57,975 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/B of 634afcda8bfed22b98df3e6e31daa427 into d563a427bee24d2b8f697baa3b5774fe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:57,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:57,975 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/B, priority=12, startTime=1731521577883; duration=0sec 2024-11-13T18:12:57,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:57,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:B 2024-11-13T18:12:57,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:12:57,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742059_1235 (size=4469) 2024-11-13T18:12:57,978 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:12:57,978 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/C is initiating minor compaction (all files) 2024-11-13T18:12:57,978 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/C in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:57,978 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/940de46e5d6a47d5bfd434cc23da26c7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5522c174ca8148a8963ce5d194f91b43, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/7161444f99024ea6bd4e8a2c5610e27f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/41a78edbe609450da18e1bc4f4d94916] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=48.8 K 2024-11-13T18:12:57,978 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#A#compaction#193 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:57,979 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 940de46e5d6a47d5bfd434cc23da26c7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731521574275 2024-11-13T18:12:57,979 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 5522c174ca8148a8963ce5d194f91b43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731521575433 2024-11-13T18:12:57,979 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 7161444f99024ea6bd4e8a2c5610e27f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731521576708 2024-11-13T18:12:57,980 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a78edbe609450da18e1bc4f4d94916, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521577483 2024-11-13T18:12:57,980 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/caf78f1642b445dc94be986953c6f043 is 175, key is test_row_0/A:col10/1731521577495/Put/seqid=0 2024-11-13T18:12:57,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:57,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:57,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:57,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:57,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:57,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:57,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f16550f4447249729061c1123eff08d9_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521577642/Put/seqid=0 2024-11-13T18:12:58,009 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#C#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:58,009 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/e729d478df8d46cd873f733b6d70218a is 50, key is test_row_0/C:col10/1731521577495/Put/seqid=0 2024-11-13T18:12:58,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742060_1236 (size=32141) 2024-11-13T18:12:58,041 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/caf78f1642b445dc94be986953c6f043 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/caf78f1642b445dc94be986953c6f043 2024-11-13T18:12:58,053 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/A of 634afcda8bfed22b98df3e6e31daa427 into caf78f1642b445dc94be986953c6f043(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:58,053 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:58,053 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/A, priority=12, startTime=1731521577883; duration=0sec 2024-11-13T18:12:58,053 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:58,053 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:A 2024-11-13T18:12:58,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521638059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521638059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521638059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742061_1237 (size=14994) 2024-11-13T18:12:58,075 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:58,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f16550f4447249729061c1123eff08d9_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f16550f4447249729061c1123eff08d9_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:58,080 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0f2158cb6ea94d45910b0842d7e26753, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:58,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0f2158cb6ea94d45910b0842d7e26753 is 175, key is test_row_0/A:col10/1731521577642/Put/seqid=0 2024-11-13T18:12:58,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742062_1238 (size=13187) 2024-11-13T18:12:58,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742063_1239 (size=39949) 2024-11-13T18:12:58,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:58,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:58,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:12:58,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521638175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521638175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521638177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521638384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521638385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521638404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:58,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:58,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:12:58,520 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/e729d478df8d46cd873f733b6d70218a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e729d478df8d46cd873f733b6d70218a 2024-11-13T18:12:58,522 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0f2158cb6ea94d45910b0842d7e26753 2024-11-13T18:12:58,540 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/C of 634afcda8bfed22b98df3e6e31daa427 into e729d478df8d46cd873f733b6d70218a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:12:58,540 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:58,540 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/C, priority=12, startTime=1731521577883; duration=0sec 2024-11-13T18:12:58,540 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:58,540 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:C 2024-11-13T18:12:58,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/2840f300e81d4c54acb5686371b0db07 is 50, key is test_row_0/B:col10/1731521577642/Put/seqid=0 2024-11-13T18:12:58,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742064_1240 (size=12301) 2024-11-13T18:12:58,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/2840f300e81d4c54acb5686371b0db07 2024-11-13T18:12:58,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/122c16778721426dbb495342299783d6 is 50, key is test_row_0/C:col10/1731521577642/Put/seqid=0 2024-11-13T18:12:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742065_1241 (size=12301) 2024-11-13T18:12:58,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/122c16778721426dbb495342299783d6 2024-11-13T18:12:58,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/0f2158cb6ea94d45910b0842d7e26753 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0f2158cb6ea94d45910b0842d7e26753 2024-11-13T18:12:58,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0f2158cb6ea94d45910b0842d7e26753, entries=200, sequenceid=386, filesize=39.0 K 2024-11-13T18:12:58,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/2840f300e81d4c54acb5686371b0db07 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2840f300e81d4c54acb5686371b0db07 2024-11-13T18:12:58,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2840f300e81d4c54acb5686371b0db07, entries=150, sequenceid=386, filesize=12.0 K 2024-11-13T18:12:58,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/122c16778721426dbb495342299783d6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/122c16778721426dbb495342299783d6 2024-11-13T18:12:58,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/122c16778721426dbb495342299783d6, entries=150, sequenceid=386, filesize=12.0 K 2024-11-13T18:12:58,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 634afcda8bfed22b98df3e6e31daa427 in 719ms, sequenceid=386, compaction requested=false 2024-11-13T18:12:58,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:58,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521578047, max timestamp=1731521578698Num uniques 859; 2024-11-13T18:12:58,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:12:58,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521578047, max timestamp=1731521578698Num uniques 859; 2024-11-13T18:12:58,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:12:58,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521578047, max timestamp=1731521578698Num uniques 859; 2024-11-13T18:12:58,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:12:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:58,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-13T18:12:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:58,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130de0f5b68d114395b15f861d137e4d52_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521578715/Put/seqid=0 2024-11-13T18:12:58,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:58,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:58,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742066_1242 (size=12454) 2024-11-13T18:12:58,795 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:58,800 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130de0f5b68d114395b15f861d137e4d52_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130de0f5b68d114395b15f861d137e4d52_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:58,801 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/587e68ced3b54ce1aa2f4acaa6f45302, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:58,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/587e68ced3b54ce1aa2f4acaa6f45302 is 175, key is test_row_0/A:col10/1731521578715/Put/seqid=0 2024-11-13T18:12:58,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521578789, max timestamp=1731521578809Num uniques 859; 2024-11-13T18:12:58,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:12:58,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521578789, max timestamp=1731521578809Num uniques 859; 2024-11-13T18:12:58,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:12:58,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521578789, max timestamp=1731521578809Num uniques 859; 2024-11-13T18:12:58,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:12:58,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742067_1243 (size=31255) 2024-11-13T18:12:58,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521638871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521638872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521638875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37026 deadline: 1731521638894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,905 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 634afcda8bfed22b98df3e6e31daa427 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18385 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:58,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37034 deadline: 1731521638914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:58,919 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_1' in 634afcda8bfed22b98df3e6e31daa427 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18399 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:12:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:58,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:58,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:12:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521638995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:59,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521638995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:59,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521638996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:59,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:59,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:59,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:59,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:59,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:59,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,236 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/587e68ced3b54ce1aa2f4acaa6f45302 2024-11-13T18:12:59,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/6848894727b947ada96e6cb10a6f749d is 50, key is test_row_0/B:col10/1731521578715/Put/seqid=0 2024-11-13T18:12:59,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742068_1244 (size=12301) 2024-11-13T18:12:59,381 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-13T18:12:59,381 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:59,381 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:59,382 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:59,382 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:59,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:59,384 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-13T18:12:59,384 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-13T18:12:59,385 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:59,385 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:59,385 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:59,385 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:59,385 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:59,385 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:59,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:59,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,389 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-13T18:12:59,389 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:12:59,389 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:59,389 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:12:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:12:59,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37074 deadline: 1731521639538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:59,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37046 deadline: 1731521639541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:59,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:12:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37086 deadline: 1731521639541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:12:59,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:59,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:59,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:59,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:59,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:12:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:12:59,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:12:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:12:59,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/6848894727b947ada96e6cb10a6f749d 2024-11-13T18:12:59,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/5767f47ef7c448ae88e63fa607f02d6a is 50, key is test_row_0/C:col10/1731521578715/Put/seqid=0 2024-11-13T18:12:59,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742069_1245 (size=12301) 2024-11-13T18:12:59,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/5767f47ef7c448ae88e63fa607f02d6a 2024-11-13T18:12:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/587e68ced3b54ce1aa2f4acaa6f45302 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/587e68ced3b54ce1aa2f4acaa6f45302 2024-11-13T18:12:59,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/587e68ced3b54ce1aa2f4acaa6f45302, entries=150, sequenceid=414, filesize=30.5 K 2024-11-13T18:12:59,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/6848894727b947ada96e6cb10a6f749d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/6848894727b947ada96e6cb10a6f749d 2024-11-13T18:12:59,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/6848894727b947ada96e6cb10a6f749d, entries=150, sequenceid=414, filesize=12.0 K 2024-11-13T18:12:59,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/5767f47ef7c448ae88e63fa607f02d6a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5767f47ef7c448ae88e63fa607f02d6a 2024-11-13T18:12:59,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5767f47ef7c448ae88e63fa607f02d6a, entries=150, sequenceid=414, filesize=12.0 K 2024-11-13T18:12:59,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 634afcda8bfed22b98df3e6e31daa427 in 1075ms, sequenceid=414, compaction requested=true 2024-11-13T18:12:59,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:12:59,796 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:59,798 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:59,798 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/A is initiating minor compaction (all files) 2024-11-13T18:12:59,798 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/A in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,798 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/caf78f1642b445dc94be986953c6f043, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0f2158cb6ea94d45910b0842d7e26753, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/587e68ced3b54ce1aa2f4acaa6f45302] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=100.9 K 2024-11-13T18:12:59,798 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,798 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/caf78f1642b445dc94be986953c6f043, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0f2158cb6ea94d45910b0842d7e26753, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/587e68ced3b54ce1aa2f4acaa6f45302] 2024-11-13T18:12:59,799 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting caf78f1642b445dc94be986953c6f043, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521577483 2024-11-13T18:12:59,800 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f2158cb6ea94d45910b0842d7e26753, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731521577632 2024-11-13T18:12:59,800 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 587e68ced3b54ce1aa2f4acaa6f45302, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731521578712 2024-11-13T18:12:59,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:12:59,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:59,818 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:12:59,819 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:12:59,819 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/B is initiating minor compaction (all files) 2024-11-13T18:12:59,819 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/B in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,819 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d563a427bee24d2b8f697baa3b5774fe, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2840f300e81d4c54acb5686371b0db07, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/6848894727b947ada96e6cb10a6f749d] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=36.9 K 2024-11-13T18:12:59,820 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d563a427bee24d2b8f697baa3b5774fe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521577483 2024-11-13T18:12:59,821 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 2840f300e81d4c54acb5686371b0db07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731521577632 2024-11-13T18:12:59,821 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6848894727b947ada96e6cb10a6f749d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731521578712 2024-11-13T18:12:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:12:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:12:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634afcda8bfed22b98df3e6e31daa427:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:12:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:12:59,837 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:59,842 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#B#compaction#202 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:59,843 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/081cb744ca4d4b0c95d5cdb73cecaec6 is 50, key is test_row_0/B:col10/1731521578715/Put/seqid=0 2024-11-13T18:12:59,849 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138718389e0d2746c9a21c9066ae8184d7_634afcda8bfed22b98df3e6e31daa427 store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:59,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:12:59,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:12:59,876 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T18:12:59,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:12:59,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:12:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:12:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:12:59,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742070_1246 (size=13289) 2024-11-13T18:12:59,894 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138718389e0d2746c9a21c9066ae8184d7_634afcda8bfed22b98df3e6e31daa427, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:59,894 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138718389e0d2746c9a21c9066ae8184d7_634afcda8bfed22b98df3e6e31daa427 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:59,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113140442490c2e4e95a59052af6f7a286c_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521578873/Put/seqid=0 2024-11-13T18:12:59,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742071_1247 (size=4469) 2024-11-13T18:12:59,934 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#A#compaction#201 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:12:59,935 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/35b14c923e7e44df8b69cada0e283f77 is 175, key is test_row_0/A:col10/1731521578715/Put/seqid=0 2024-11-13T18:12:59,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742072_1248 (size=12454) 2024-11-13T18:12:59,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:12:59,969 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113140442490c2e4e95a59052af6f7a286c_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113140442490c2e4e95a59052af6f7a286c_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:12:59,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/c10ee909d6c445448b75a5f4a5563c4f, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:12:59,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/c10ee909d6c445448b75a5f4a5563c4f is 175, key is test_row_0/A:col10/1731521578873/Put/seqid=0 2024-11-13T18:12:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:12:59,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742073_1249 (size=32243) 2024-11-13T18:13:00,000 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/35b14c923e7e44df8b69cada0e283f77 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/35b14c923e7e44df8b69cada0e283f77 2024-11-13T18:13:00,010 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/A of 634afcda8bfed22b98df3e6e31daa427 into 35b14c923e7e44df8b69cada0e283f77(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:00,010 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:13:00,010 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/A, priority=13, startTime=1731521579796; duration=0sec 2024-11-13T18:13:00,010 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:00,011 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:A 2024-11-13T18:13:00,011 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:00,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742074_1250 (size=31255) 2024-11-13T18:13:00,012 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=450, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/c10ee909d6c445448b75a5f4a5563c4f 2024-11-13T18:13:00,013 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:00,013 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 634afcda8bfed22b98df3e6e31daa427/C is initiating minor compaction (all files) 2024-11-13T18:13:00,013 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634afcda8bfed22b98df3e6e31daa427/C in TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:13:00,013 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e729d478df8d46cd873f733b6d70218a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/122c16778721426dbb495342299783d6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5767f47ef7c448ae88e63fa607f02d6a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp, totalSize=36.9 K 2024-11-13T18:13:00,014 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e729d478df8d46cd873f733b6d70218a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521577483 2024-11-13T18:13:00,015 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 122c16778721426dbb495342299783d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731521577632 2024-11-13T18:13:00,016 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5767f47ef7c448ae88e63fa607f02d6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731521578712 2024-11-13T18:13:00,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/4362cbace9ba47528d5821b964a90a57 is 50, key is test_row_0/B:col10/1731521578873/Put/seqid=0 2024-11-13T18:13:00,043 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634afcda8bfed22b98df3e6e31daa427#C#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:00,044 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/9e1069ccdde54c13a74ea65c564da6d4 is 50, key is test_row_0/C:col10/1731521578715/Put/seqid=0 2024-11-13T18:13:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:00,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. as already flushing 2024-11-13T18:13:00,061 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-13T18:13:00,061 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:00,061 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:00,061 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-13T18:13:00,061 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:00,061 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:00,061 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:00,061 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:00,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:00,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:00,070 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-13T18:13:00,070 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:00,070 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:00,070 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:00,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:00,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742075_1251 (size=12301) 2024-11-13T18:13:00,128 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/4362cbace9ba47528d5821b964a90a57 2024-11-13T18:13:00,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742076_1252 (size=13289) 2024-11-13T18:13:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/0519874c96c34040bc93659ad843f22c is 50, key is test_row_0/C:col10/1731521578873/Put/seqid=0 2024-11-13T18:13:00,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742077_1253 (size=12301) 2024-11-13T18:13:00,236 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/0519874c96c34040bc93659ad843f22c 2024-11-13T18:13:00,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/c10ee909d6c445448b75a5f4a5563c4f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c10ee909d6c445448b75a5f4a5563c4f 2024-11-13T18:13:00,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c10ee909d6c445448b75a5f4a5563c4f, entries=150, sequenceid=450, filesize=30.5 K 2024-11-13T18:13:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/4362cbace9ba47528d5821b964a90a57 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4362cbace9ba47528d5821b964a90a57 2024-11-13T18:13:00,278 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4362cbace9ba47528d5821b964a90a57, entries=150, sequenceid=450, filesize=12.0 K 2024-11-13T18:13:00,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/0519874c96c34040bc93659ad843f22c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/0519874c96c34040bc93659ad843f22c 2024-11-13T18:13:00,296 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/081cb744ca4d4b0c95d5cdb73cecaec6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/081cb744ca4d4b0c95d5cdb73cecaec6 2024-11-13T18:13:00,299 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/0519874c96c34040bc93659ad843f22c, entries=150, sequenceid=450, filesize=12.0 K 2024-11-13T18:13:00,308 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=20.13 KB/20610 for 634afcda8bfed22b98df3e6e31daa427 in 431ms, sequenceid=450, compaction requested=false 2024-11-13T18:13:00,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:13:00,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:13:00,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-13T18:13:00,308 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/B of 634afcda8bfed22b98df3e6e31daa427 into 081cb744ca4d4b0c95d5cdb73cecaec6(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:00,309 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:13:00,309 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/B, priority=13, startTime=1731521579817; duration=0sec 2024-11-13T18:13:00,309 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:00,309 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:B 2024-11-13T18:13:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-13T18:13:00,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-13T18:13:00,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4830 sec 2024-11-13T18:13:00,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.4890 sec 2024-11-13T18:13:00,540 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/9e1069ccdde54c13a74ea65c564da6d4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/9e1069ccdde54c13a74ea65c564da6d4 2024-11-13T18:13:00,551 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634afcda8bfed22b98df3e6e31daa427/C of 634afcda8bfed22b98df3e6e31daa427 into 9e1069ccdde54c13a74ea65c564da6d4(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:00,551 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634afcda8bfed22b98df3e6e31daa427: 2024-11-13T18:13:00,551 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., storeName=634afcda8bfed22b98df3e6e31daa427/C, priority=13, startTime=1731521579826; duration=0sec 2024-11-13T18:13:00,551 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:00,552 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634afcda8bfed22b98df3e6e31daa427:C 2024-11-13T18:13:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:13:01,995 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:04,984 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:13:08,926 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-13T18:13:08,927 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:08,927 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:08,927 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:08,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:08,975 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-13T18:13:08,975 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:08,975 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:08,975 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:08,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1799 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1752 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 839 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2517 rows 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 828 2024-11-13T18:13:08,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2483 rows 2024-11-13T18:13:08,976 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:13:08,976 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:13:08,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:08,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:08,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:08,977 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:13:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:13:08,981 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521588981"}]},"ts":"1731521588981"} 2024-11-13T18:13:08,983 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:13:08,983 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:13:08,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:13:08,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, UNASSIGN}] 2024-11-13T18:13:08,986 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, UNASSIGN 2024-11-13T18:13:08,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=48 updating hbase:meta row=634afcda8bfed22b98df3e6e31daa427, regionState=CLOSING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:08,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, UNASSIGN because future has completed 2024-11-13T18:13:08,988 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:13:08,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; CloseRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:13:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:13:09,141 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(122): Close 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:09,141 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:13:09,141 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1722): Closing 634afcda8bfed22b98df3e6e31daa427, disabling compactions & flushes 2024-11-13T18:13:09,141 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:13:09,141 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:13:09,141 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. after waiting 0 ms 2024-11-13T18:13:09,141 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:13:09,141 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(2902): Flushing 634afcda8bfed22b98df3e6e31daa427 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T18:13:09,142 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=A 2024-11-13T18:13:09,142 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:09,142 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=B 2024-11-13T18:13:09,142 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:09,142 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634afcda8bfed22b98df3e6e31daa427, store=C 2024-11-13T18:13:09,142 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:09,149 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133cad406b6dab47f0b5d9bf71cf5fc5c1_634afcda8bfed22b98df3e6e31daa427 is 50, key is test_row_0/A:col10/1731521588925/Put/seqid=0 2024-11-13T18:13:09,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742078_1254 (size=9914) 2024-11-13T18:13:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:13:09,554 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:09,558 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133cad406b6dab47f0b5d9bf71cf5fc5c1_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133cad406b6dab47f0b5d9bf71cf5fc5c1_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:09,559 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/351536e0f18b42ef98289eaeb9677804, store: [table=TestAcidGuarantees family=A region=634afcda8bfed22b98df3e6e31daa427] 2024-11-13T18:13:09,560 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/351536e0f18b42ef98289eaeb9677804 is 175, key is test_row_0/A:col10/1731521588925/Put/seqid=0 2024-11-13T18:13:09,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742079_1255 (size=22561) 2024-11-13T18:13:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:13:09,970 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=461, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/351536e0f18b42ef98289eaeb9677804 2024-11-13T18:13:09,977 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/95577ee7d6e44c9d98c73d4236f74c63 is 50, key is test_row_0/B:col10/1731521588925/Put/seqid=0 2024-11-13T18:13:09,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742080_1256 (size=9857) 2024-11-13T18:13:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:13:10,383 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/95577ee7d6e44c9d98c73d4236f74c63 2024-11-13T18:13:10,390 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/56872965dcc842a98a18b6db4fbd7da2 is 50, key is test_row_0/C:col10/1731521588925/Put/seqid=0 2024-11-13T18:13:10,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742081_1257 (size=9857) 2024-11-13T18:13:10,796 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/56872965dcc842a98a18b6db4fbd7da2 2024-11-13T18:13:10,801 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/A/351536e0f18b42ef98289eaeb9677804 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/351536e0f18b42ef98289eaeb9677804 2024-11-13T18:13:10,805 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/351536e0f18b42ef98289eaeb9677804, entries=100, sequenceid=461, filesize=22.0 K 2024-11-13T18:13:10,806 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/B/95577ee7d6e44c9d98c73d4236f74c63 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/95577ee7d6e44c9d98c73d4236f74c63 2024-11-13T18:13:10,809 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/95577ee7d6e44c9d98c73d4236f74c63, entries=100, sequenceid=461, filesize=9.6 K 2024-11-13T18:13:10,810 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/.tmp/C/56872965dcc842a98a18b6db4fbd7da2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/56872965dcc842a98a18b6db4fbd7da2 2024-11-13T18:13:10,813 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/56872965dcc842a98a18b6db4fbd7da2, entries=100, sequenceid=461, filesize=9.6 K 2024-11-13T18:13:10,814 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 634afcda8bfed22b98df3e6e31daa427 in 1673ms, sequenceid=461, compaction requested=true 2024-11-13T18:13:10,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/02eed6c2dec245fbb2648c43b362405d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a691946e1c80495181a55e0a3104a2e9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/889a7e7d5c574fafa7ba7c2c68ed03ce, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5cc7d5f95854439bb1e14252e503d977, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/ee6892e17bbd4a75a3b76ef502e15cfa, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/88c659aa60304bf2874f4cb95c7e4f92, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/1c642e46e4c94a578bf5a97ceba34a3b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/91f41669a1994fd1a4b449bb813c56ce, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/78f570390823416fbcc315e157ca14a9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3e5f77c31c654b8bae5b50ca10f3d9fb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0215a19cf8324a70b913940bb7c3f922, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a4bb4aed96a44c619fc80092e772a8a7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5d44869ed8d245b1897e4943b06c1d03, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/cedcc557a97347fbb65bd0253187373d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0b9e93bd72514f09917ea0aac6869976, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3f72e055542d406cab4c137f8117aba8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/622a5d79af2c4f16828a4e841f5df598, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/08c888e09f4b40a0ae83db769ee9f578, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/221c854b95b34ee7a36aa383530d3ad7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5f932b5804e447a972ecfa1a3977dda, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/47f87cb160514b1e9fb9db1aaf3d27c5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/f4eef13adebe4fe195856b9f307b2cc6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6f1e02a63bf74ee79e41168595f24d31, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5fc7fd50c40442d9100a0ba016c3e2c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6d1579bebc7749b7a715a2e7fa1a8d0a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/caf78f1642b445dc94be986953c6f043, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0f2158cb6ea94d45910b0842d7e26753, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/587e68ced3b54ce1aa2f4acaa6f45302] to archive 2024-11-13T18:13:10,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:13:10,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/02eed6c2dec245fbb2648c43b362405d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/02eed6c2dec245fbb2648c43b362405d 2024-11-13T18:13:10,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a691946e1c80495181a55e0a3104a2e9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a691946e1c80495181a55e0a3104a2e9 2024-11-13T18:13:10,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/889a7e7d5c574fafa7ba7c2c68ed03ce to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/889a7e7d5c574fafa7ba7c2c68ed03ce 2024-11-13T18:13:10,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5cc7d5f95854439bb1e14252e503d977 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5cc7d5f95854439bb1e14252e503d977 2024-11-13T18:13:10,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/ee6892e17bbd4a75a3b76ef502e15cfa to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/ee6892e17bbd4a75a3b76ef502e15cfa 2024-11-13T18:13:10,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/88c659aa60304bf2874f4cb95c7e4f92 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/88c659aa60304bf2874f4cb95c7e4f92 2024-11-13T18:13:10,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/1c642e46e4c94a578bf5a97ceba34a3b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/1c642e46e4c94a578bf5a97ceba34a3b 2024-11-13T18:13:10,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/91f41669a1994fd1a4b449bb813c56ce to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/91f41669a1994fd1a4b449bb813c56ce 2024-11-13T18:13:10,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/78f570390823416fbcc315e157ca14a9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/78f570390823416fbcc315e157ca14a9 2024-11-13T18:13:10,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3e5f77c31c654b8bae5b50ca10f3d9fb to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3e5f77c31c654b8bae5b50ca10f3d9fb 2024-11-13T18:13:10,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0215a19cf8324a70b913940bb7c3f922 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0215a19cf8324a70b913940bb7c3f922 2024-11-13T18:13:10,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a4bb4aed96a44c619fc80092e772a8a7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/a4bb4aed96a44c619fc80092e772a8a7 2024-11-13T18:13:10,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5d44869ed8d245b1897e4943b06c1d03 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5d44869ed8d245b1897e4943b06c1d03 2024-11-13T18:13:10,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/cedcc557a97347fbb65bd0253187373d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/cedcc557a97347fbb65bd0253187373d 2024-11-13T18:13:10,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0b9e93bd72514f09917ea0aac6869976 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0b9e93bd72514f09917ea0aac6869976 2024-11-13T18:13:10,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3f72e055542d406cab4c137f8117aba8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/3f72e055542d406cab4c137f8117aba8 2024-11-13T18:13:10,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/622a5d79af2c4f16828a4e841f5df598 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/622a5d79af2c4f16828a4e841f5df598 2024-11-13T18:13:10,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/08c888e09f4b40a0ae83db769ee9f578 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/08c888e09f4b40a0ae83db769ee9f578 2024-11-13T18:13:10,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/221c854b95b34ee7a36aa383530d3ad7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/221c854b95b34ee7a36aa383530d3ad7 2024-11-13T18:13:10,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5f932b5804e447a972ecfa1a3977dda to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5f932b5804e447a972ecfa1a3977dda 2024-11-13T18:13:10,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/47f87cb160514b1e9fb9db1aaf3d27c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/47f87cb160514b1e9fb9db1aaf3d27c5 2024-11-13T18:13:10,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/f4eef13adebe4fe195856b9f307b2cc6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/f4eef13adebe4fe195856b9f307b2cc6 2024-11-13T18:13:10,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6f1e02a63bf74ee79e41168595f24d31 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6f1e02a63bf74ee79e41168595f24d31 2024-11-13T18:13:10,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5fc7fd50c40442d9100a0ba016c3e2c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c5fc7fd50c40442d9100a0ba016c3e2c 2024-11-13T18:13:10,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6d1579bebc7749b7a715a2e7fa1a8d0a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/6d1579bebc7749b7a715a2e7fa1a8d0a 2024-11-13T18:13:10,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/5e1e7f3b71d7492bbfac24f5bb6bb7c0 2024-11-13T18:13:10,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/caf78f1642b445dc94be986953c6f043 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/caf78f1642b445dc94be986953c6f043 2024-11-13T18:13:10,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0f2158cb6ea94d45910b0842d7e26753 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/0f2158cb6ea94d45910b0842d7e26753 2024-11-13T18:13:10,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/587e68ced3b54ce1aa2f4acaa6f45302 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/587e68ced3b54ce1aa2f4acaa6f45302 2024-11-13T18:13:10,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5eb76bf35afd4443bb23219c56744fc6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4b61681d381c413bbce43ce0be12bb36, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/34593026383f4f9c91b340cefd7b9150, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2ccbeacb88df45f5aabdaac768fc81c6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4c4e54e8f9cc4d8aaf5b15b4f5a09e8b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/494e4eacf37f460e99ea3446fdee3c4f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d939f2ca07fd49429138b090b2329edf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2c4e3377fb2c479284dd19936e0a6847, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8d6beacc789c44c39c2a98cc35bc66b4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/52029bd40ea14733b6f4e6404f21ff97, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/b61c63492cb0413793d8f6e414e62468, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/cd8ee47ce3704714a5b67403785e60f9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/81149015c65847e7a9fd5a3b68a58b9f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8e8b25cba073427bbb16d32d421492bc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/adaead4e4c6f433e9592ade4d6e611a9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/cf213bfbdfb64bdf91a9b4ebe9a9e597, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/ec1fd97bb8f44dc99f2563b6d68b574a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/96cb19197afc472584a907bfa3a40b5c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/123a7cf1fa59430ea7b0eede2950ba32, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/efc3cc5b9496475fa8ae1cbc40a28a4f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5dda75549e6e4fda91c0b986695dc54b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8a809781913f498b916c8e10771d9a10, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/665b6f0551a644e5ae637e71b2a0518e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/1e0c4b59b16e45c2b62a98e49fda8d10, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/488b1680085e4d1c97a181b4b35e7587, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d563a427bee24d2b8f697baa3b5774fe, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/f9e917dfc52849f08a278e13fbadc5d8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2840f300e81d4c54acb5686371b0db07, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/6848894727b947ada96e6cb10a6f749d] to archive 2024-11-13T18:13:10,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:13:10,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5eb76bf35afd4443bb23219c56744fc6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5eb76bf35afd4443bb23219c56744fc6 2024-11-13T18:13:10,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4b61681d381c413bbce43ce0be12bb36 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4b61681d381c413bbce43ce0be12bb36 2024-11-13T18:13:10,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/34593026383f4f9c91b340cefd7b9150 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/34593026383f4f9c91b340cefd7b9150 2024-11-13T18:13:10,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2ccbeacb88df45f5aabdaac768fc81c6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2ccbeacb88df45f5aabdaac768fc81c6 2024-11-13T18:13:10,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4c4e54e8f9cc4d8aaf5b15b4f5a09e8b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4c4e54e8f9cc4d8aaf5b15b4f5a09e8b 2024-11-13T18:13:10,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/494e4eacf37f460e99ea3446fdee3c4f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/494e4eacf37f460e99ea3446fdee3c4f 2024-11-13T18:13:10,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d939f2ca07fd49429138b090b2329edf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d939f2ca07fd49429138b090b2329edf 2024-11-13T18:13:10,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2c4e3377fb2c479284dd19936e0a6847 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2c4e3377fb2c479284dd19936e0a6847 2024-11-13T18:13:10,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8d6beacc789c44c39c2a98cc35bc66b4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8d6beacc789c44c39c2a98cc35bc66b4 2024-11-13T18:13:10,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/52029bd40ea14733b6f4e6404f21ff97 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/52029bd40ea14733b6f4e6404f21ff97 2024-11-13T18:13:10,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/b61c63492cb0413793d8f6e414e62468 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/b61c63492cb0413793d8f6e414e62468 2024-11-13T18:13:10,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/cd8ee47ce3704714a5b67403785e60f9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/cd8ee47ce3704714a5b67403785e60f9 2024-11-13T18:13:10,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/81149015c65847e7a9fd5a3b68a58b9f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/81149015c65847e7a9fd5a3b68a58b9f 2024-11-13T18:13:10,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8e8b25cba073427bbb16d32d421492bc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8e8b25cba073427bbb16d32d421492bc 2024-11-13T18:13:10,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/adaead4e4c6f433e9592ade4d6e611a9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/adaead4e4c6f433e9592ade4d6e611a9 2024-11-13T18:13:10,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/cf213bfbdfb64bdf91a9b4ebe9a9e597 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/cf213bfbdfb64bdf91a9b4ebe9a9e597 2024-11-13T18:13:10,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/ec1fd97bb8f44dc99f2563b6d68b574a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/ec1fd97bb8f44dc99f2563b6d68b574a 2024-11-13T18:13:10,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/96cb19197afc472584a907bfa3a40b5c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/96cb19197afc472584a907bfa3a40b5c 2024-11-13T18:13:10,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/123a7cf1fa59430ea7b0eede2950ba32 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/123a7cf1fa59430ea7b0eede2950ba32 2024-11-13T18:13:10,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/efc3cc5b9496475fa8ae1cbc40a28a4f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/efc3cc5b9496475fa8ae1cbc40a28a4f 2024-11-13T18:13:10,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5dda75549e6e4fda91c0b986695dc54b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/5dda75549e6e4fda91c0b986695dc54b 2024-11-13T18:13:10,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8a809781913f498b916c8e10771d9a10 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/8a809781913f498b916c8e10771d9a10 2024-11-13T18:13:10,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/665b6f0551a644e5ae637e71b2a0518e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/665b6f0551a644e5ae637e71b2a0518e 2024-11-13T18:13:10,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/1e0c4b59b16e45c2b62a98e49fda8d10 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/1e0c4b59b16e45c2b62a98e49fda8d10 2024-11-13T18:13:10,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/488b1680085e4d1c97a181b4b35e7587 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/488b1680085e4d1c97a181b4b35e7587 2024-11-13T18:13:10,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d563a427bee24d2b8f697baa3b5774fe to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/d563a427bee24d2b8f697baa3b5774fe 2024-11-13T18:13:10,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/f9e917dfc52849f08a278e13fbadc5d8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/f9e917dfc52849f08a278e13fbadc5d8 2024-11-13T18:13:10,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2840f300e81d4c54acb5686371b0db07 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/2840f300e81d4c54acb5686371b0db07 2024-11-13T18:13:10,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/6848894727b947ada96e6cb10a6f749d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/6848894727b947ada96e6cb10a6f749d 2024-11-13T18:13:10,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e1b5e9ebe75a4502b6fee84d5bdf6096, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fa0d6c6059cc4a3cb8d831fc302d49c8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/457aeb6039744b3183060bf4abfa489f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/b96a2ecb6e39476c9f85c2ef0d7c61dc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/28d4a095c82c4e02b264f560f57d5ef2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/c008875b8ac94e4bac1828a4393ce9ba, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d505c863518b4c93b5974fffc5c2381a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/12efb45a9be94e0388d4089f5702abf1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e011952f6105410d873434dd0fbebe49, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/9be3decddb5a4ca7b9e9a37cdc18e607, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dee6760498ab4ac3a50d2e963de91a34, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/8d5a264507614b379402d89c625b2c2b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/77dfa805bb8a4ddeb2a412cd4c4a72cd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/104b56771dea41088df9e45c0d5df2aa, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/ff28d351c1ff41019a1380d753cf9753, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/07c8dacec2294df189a3342dec3ead6e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dad1f25c11264ea2848bcb6d4f18b7a2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/4d751164b588441da677cc3012620266, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fe435e12d24c41e5a589e9455380a733, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d2a2850a86bd4653bd6770842ae6459f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/20e0def1ba144b74a61937589d436549, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/940de46e5d6a47d5bfd434cc23da26c7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/147f9f3d1ec34a3cbf5bbf419fd708c6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5522c174ca8148a8963ce5d194f91b43, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/7161444f99024ea6bd4e8a2c5610e27f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e729d478df8d46cd873f733b6d70218a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/41a78edbe609450da18e1bc4f4d94916, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/122c16778721426dbb495342299783d6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5767f47ef7c448ae88e63fa607f02d6a] to archive 2024-11-13T18:13:10,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:13:10,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e1b5e9ebe75a4502b6fee84d5bdf6096 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e1b5e9ebe75a4502b6fee84d5bdf6096 2024-11-13T18:13:10,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fa0d6c6059cc4a3cb8d831fc302d49c8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fa0d6c6059cc4a3cb8d831fc302d49c8 2024-11-13T18:13:10,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/457aeb6039744b3183060bf4abfa489f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/457aeb6039744b3183060bf4abfa489f 2024-11-13T18:13:10,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/b96a2ecb6e39476c9f85c2ef0d7c61dc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/b96a2ecb6e39476c9f85c2ef0d7c61dc 2024-11-13T18:13:10,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/28d4a095c82c4e02b264f560f57d5ef2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/28d4a095c82c4e02b264f560f57d5ef2 2024-11-13T18:13:10,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/c008875b8ac94e4bac1828a4393ce9ba to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/c008875b8ac94e4bac1828a4393ce9ba 2024-11-13T18:13:10,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d505c863518b4c93b5974fffc5c2381a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d505c863518b4c93b5974fffc5c2381a 2024-11-13T18:13:10,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/12efb45a9be94e0388d4089f5702abf1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/12efb45a9be94e0388d4089f5702abf1 2024-11-13T18:13:10,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e011952f6105410d873434dd0fbebe49 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e011952f6105410d873434dd0fbebe49 2024-11-13T18:13:10,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/9be3decddb5a4ca7b9e9a37cdc18e607 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/9be3decddb5a4ca7b9e9a37cdc18e607 2024-11-13T18:13:10,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dee6760498ab4ac3a50d2e963de91a34 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dee6760498ab4ac3a50d2e963de91a34 2024-11-13T18:13:10,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/8d5a264507614b379402d89c625b2c2b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/8d5a264507614b379402d89c625b2c2b 2024-11-13T18:13:10,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/77dfa805bb8a4ddeb2a412cd4c4a72cd to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/77dfa805bb8a4ddeb2a412cd4c4a72cd 2024-11-13T18:13:10,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/104b56771dea41088df9e45c0d5df2aa to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/104b56771dea41088df9e45c0d5df2aa 2024-11-13T18:13:10,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/ff28d351c1ff41019a1380d753cf9753 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/ff28d351c1ff41019a1380d753cf9753 2024-11-13T18:13:10,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/07c8dacec2294df189a3342dec3ead6e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/07c8dacec2294df189a3342dec3ead6e 2024-11-13T18:13:10,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dad1f25c11264ea2848bcb6d4f18b7a2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/dad1f25c11264ea2848bcb6d4f18b7a2 2024-11-13T18:13:10,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/4d751164b588441da677cc3012620266 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/4d751164b588441da677cc3012620266 2024-11-13T18:13:10,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fe435e12d24c41e5a589e9455380a733 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/fe435e12d24c41e5a589e9455380a733 2024-11-13T18:13:10,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d2a2850a86bd4653bd6770842ae6459f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/d2a2850a86bd4653bd6770842ae6459f 2024-11-13T18:13:10,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/20e0def1ba144b74a61937589d436549 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/20e0def1ba144b74a61937589d436549 2024-11-13T18:13:10,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/940de46e5d6a47d5bfd434cc23da26c7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/940de46e5d6a47d5bfd434cc23da26c7 2024-11-13T18:13:10,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/147f9f3d1ec34a3cbf5bbf419fd708c6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/147f9f3d1ec34a3cbf5bbf419fd708c6 2024-11-13T18:13:10,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5522c174ca8148a8963ce5d194f91b43 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5522c174ca8148a8963ce5d194f91b43 2024-11-13T18:13:10,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/7161444f99024ea6bd4e8a2c5610e27f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/7161444f99024ea6bd4e8a2c5610e27f 2024-11-13T18:13:10,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e729d478df8d46cd873f733b6d70218a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/e729d478df8d46cd873f733b6d70218a 2024-11-13T18:13:10,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/41a78edbe609450da18e1bc4f4d94916 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/41a78edbe609450da18e1bc4f4d94916 2024-11-13T18:13:10,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/122c16778721426dbb495342299783d6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/122c16778721426dbb495342299783d6 2024-11-13T18:13:10,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5767f47ef7c448ae88e63fa607f02d6a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/5767f47ef7c448ae88e63fa607f02d6a 2024-11-13T18:13:10,953 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/recovered.edits/464.seqid, newMaxSeqId=464, maxSeqId=4 2024-11-13T18:13:10,954 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. 2024-11-13T18:13:10,954 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1676): Region close journal for 634afcda8bfed22b98df3e6e31daa427: Waiting for close lock at 1731521589141Running coprocessor pre-close hooks at 1731521589141Disabling compacts and flushes for region at 1731521589141Disabling writes for close at 1731521589141Obtaining lock to block concurrent updates at 1731521589141Preparing flush snapshotting stores in 634afcda8bfed22b98df3e6e31daa427 at 1731521589141Finished memstore snapshotting TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731521589142 (+1 ms)Flushing stores of TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427. at 1731521589143 (+1 ms)Flushing 634afcda8bfed22b98df3e6e31daa427/A: creating writer at 1731521589143Flushing mob file 634afcda8bfed22b98df3e6e31daa427/A: appending metadata at 1731521589149 (+6 ms)Flushing mob file 634afcda8bfed22b98df3e6e31daa427/A: closing flushed file at 1731521589149Flushing 634afcda8bfed22b98df3e6e31daa427/A: appending metadata at 1731521589559 (+410 ms)Flushing 634afcda8bfed22b98df3e6e31daa427/A: closing flushed file at 1731521589560 (+1 ms)Flushing 634afcda8bfed22b98df3e6e31daa427/B: creating writer at 1731521589974 (+414 ms)Flushing 634afcda8bfed22b98df3e6e31daa427/B: appending metadata at 1731521589977 (+3 ms)Flushing 634afcda8bfed22b98df3e6e31daa427/B: closing flushed file at 1731521589977Flushing 634afcda8bfed22b98df3e6e31daa427/C: creating writer at 1731521590387 (+410 ms)Flushing 634afcda8bfed22b98df3e6e31daa427/C: appending metadata at 1731521590390 (+3 ms)Flushing 634afcda8bfed22b98df3e6e31daa427/C: closing flushed file at 1731521590390Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b76a47c: reopening flushed file at 1731521590800 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70dc290d: reopening flushed file at 1731521590805 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69142331: reopening flushed file at 1731521590809 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 634afcda8bfed22b98df3e6e31daa427 in 1673ms, sequenceid=461, compaction requested=true at 1731521590814 (+5 ms)Writing region close event to WAL at 1731521590945 (+131 ms)Running coprocessor post-close hooks at 1731521590954 (+9 ms)Closed at 1731521590954 2024-11-13T18:13:10,957 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(157): Closed 634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:10,958 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=48 updating hbase:meta row=634afcda8bfed22b98df3e6e31daa427, regionState=CLOSED 2024-11-13T18:13:10,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=48, state=RUNNABLE, hasLock=false; CloseRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:13:10,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-13T18:13:10,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseRegionProcedure 634afcda8bfed22b98df3e6e31daa427, server=fda96cf025dc,44483,1731521526892 in 1.9730 sec 2024-11-13T18:13:10,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-13T18:13:10,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634afcda8bfed22b98df3e6e31daa427, UNASSIGN in 1.9800 sec 2024-11-13T18:13:10,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T18:13:10,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9860 sec 2024-11-13T18:13:10,975 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521590974"}]},"ts":"1731521590974"} 2024-11-13T18:13:10,977 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:13:10,977 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:13:10,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0010 sec 2024-11-13T18:13:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:13:11,124 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:11,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:13:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:11,127 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=50, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T18:13:11,128 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=50, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:11,131 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,133 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/recovered.edits] 2024-11-13T18:13:11,136 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/351536e0f18b42ef98289eaeb9677804 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/351536e0f18b42ef98289eaeb9677804 2024-11-13T18:13:11,137 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/35b14c923e7e44df8b69cada0e283f77 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/35b14c923e7e44df8b69cada0e283f77 2024-11-13T18:13:11,138 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c10ee909d6c445448b75a5f4a5563c4f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/A/c10ee909d6c445448b75a5f4a5563c4f 2024-11-13T18:13:11,140 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/081cb744ca4d4b0c95d5cdb73cecaec6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/081cb744ca4d4b0c95d5cdb73cecaec6 2024-11-13T18:13:11,141 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4362cbace9ba47528d5821b964a90a57 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/4362cbace9ba47528d5821b964a90a57 2024-11-13T18:13:11,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/95577ee7d6e44c9d98c73d4236f74c63 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/B/95577ee7d6e44c9d98c73d4236f74c63 2024-11-13T18:13:11,145 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/0519874c96c34040bc93659ad843f22c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/0519874c96c34040bc93659ad843f22c 2024-11-13T18:13:11,146 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/56872965dcc842a98a18b6db4fbd7da2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/56872965dcc842a98a18b6db4fbd7da2 2024-11-13T18:13:11,148 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/9e1069ccdde54c13a74ea65c564da6d4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/C/9e1069ccdde54c13a74ea65c564da6d4 2024-11-13T18:13:11,151 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/recovered.edits/464.seqid to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427/recovered.edits/464.seqid 2024-11-13T18:13:11,152 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,152 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:13:11,152 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:13:11,153 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T18:13:11,157 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304c394a5e2f246f386b4dfb4a84afeb2_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304c394a5e2f246f386b4dfb4a84afeb2_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,158 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130de0f5b68d114395b15f861d137e4d52_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130de0f5b68d114395b15f861d137e4d52_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,160 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113140442490c2e4e95a59052af6f7a286c_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113140442490c2e4e95a59052af6f7a286c_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,161 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111321d9b6a30cdb4afab46b24f6a65f7e00_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111321d9b6a30cdb4afab46b24f6a65f7e00_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,163 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132545b68df6c043f8aea8bb4e7dfb4c50_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132545b68df6c043f8aea8bb4e7dfb4c50_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,164 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111326f86c46b5b1431ea61f81eb32026d13_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111326f86c46b5b1431ea61f81eb32026d13_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,166 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c6fc83d3fca44efa34c11180e7c92f7_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c6fc83d3fca44efa34c11180e7c92f7_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,168 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133cad406b6dab47f0b5d9bf71cf5fc5c1_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133cad406b6dab47f0b5d9bf71cf5fc5c1_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,170 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133fa9170a5418412ba81a5ce5044b0f8c_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133fa9170a5418412ba81a5ce5044b0f8c_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,172 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137a537026819c466e80044f0c1aff972b_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137a537026819c466e80044f0c1aff972b_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,173 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111382decf0dfcc548b9b9ce31f2cc28621b_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111382decf0dfcc548b9b9ce31f2cc28621b_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,175 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138e0bb57474d84beea3f346d2a449a77e_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138e0bb57474d84beea3f346d2a449a77e_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,176 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113965ac7f1bc854a028d3f91af34cfdbf6_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113965ac7f1bc854a028d3f91af34cfdbf6_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,177 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d94544e4e8244c482a36a70845a232b_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d94544e4e8244c482a36a70845a232b_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,179 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4b4cae8bb834f3f952170644c1a6c5a_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4b4cae8bb834f3f952170644c1a6c5a_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,180 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ac95f2957c8443caaa4bfeba620a04f9_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ac95f2957c8443caaa4bfeba620a04f9_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,181 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b3ae6a2a42d6412c9239975617abbded_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b3ae6a2a42d6412c9239975617abbded_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,182 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ce24d5eb84f1429484acde1e37074a1c_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ce24d5eb84f1429484acde1e37074a1c_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,184 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e158012ac00a47e1b7646c8b794e4358_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e158012ac00a47e1b7646c8b794e4358_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,185 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e701dd56b669472aa9f709cb5bbb3827_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e701dd56b669472aa9f709cb5bbb3827_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,186 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f16550f4447249729061c1123eff08d9_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f16550f4447249729061c1123eff08d9_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,188 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f88063df52124720bc14fd5ab09088ab_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f88063df52124720bc14fd5ab09088ab_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,190 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f8bce870584d431ab8eb23799bded1f7_634afcda8bfed22b98df3e6e31daa427 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f8bce870584d431ab8eb23799bded1f7_634afcda8bfed22b98df3e6e31daa427 2024-11-13T18:13:11,190 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:13:11,192 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=50, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:11,195 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:13:11,197 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:13:11,199 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=50, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:11,199 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:13:11,200 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731521591199"}]},"ts":"9223372036854775807"} 2024-11-13T18:13:11,202 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:13:11,202 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 634afcda8bfed22b98df3e6e31daa427, NAME => 'TestAcidGuarantees,,1731521557507.634afcda8bfed22b98df3e6e31daa427.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:13:11,202 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:13:11,203 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731521591202"}]},"ts":"9223372036854775807"} 2024-11-13T18:13:11,205 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:13:11,206 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=50, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:11,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-13T18:13:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T18:13:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:13:11,234 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:11,246 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/cluster_2b3eda0b-9365-a4bb-f234-af4e8c2937bc/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/cluster_2b3eda0b-9365-a4bb-f234-af4e8c2937bc/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1307743371_22 at /127.0.0.1:51632 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=663 (was 677), ProcessCount=11 (was 11), AvailableMemoryMB=810 (was 1025) 2024-11-13T18:13:11,255 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=663, ProcessCount=11, AvailableMemoryMB=809 2024-11-13T18:13:11,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:13:11,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:13:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:11,258 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:13:11,258 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:11,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 51 2024-11-13T18:13:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T18:13:11,259 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:13:11,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742082_1258 (size=1061) 2024-11-13T18:13:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T18:13:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T18:13:11,670 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1 2024-11-13T18:13:11,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742083_1259 (size=53) 2024-11-13T18:13:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T18:13:12,086 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:13:12,086 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5c8c2fb53a8f19068c501d6951c56b68, disabling compactions & flushes 2024-11-13T18:13:12,086 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:12,086 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:12,086 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. after waiting 0 ms 2024-11-13T18:13:12,086 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:12,086 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:12,086 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5c8c2fb53a8f19068c501d6951c56b68: Waiting for close lock at 1731521592086Disabling compacts and flushes for region at 1731521592086Disabling writes for close at 1731521592086Writing region close event to WAL at 1731521592086Closed at 1731521592086 2024-11-13T18:13:12,088 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:13:12,088 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731521592088"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521592088"}]},"ts":"1731521592088"} 2024-11-13T18:13:12,090 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:13:12,091 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:13:12,091 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521592091"}]},"ts":"1731521592091"} 2024-11-13T18:13:12,092 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:13:12,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c8c2fb53a8f19068c501d6951c56b68, ASSIGN}] 2024-11-13T18:13:12,094 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c8c2fb53a8f19068c501d6951c56b68, ASSIGN 2024-11-13T18:13:12,095 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c8c2fb53a8f19068c501d6951c56b68, ASSIGN; state=OFFLINE, location=fda96cf025dc,44483,1731521526892; forceNewPlan=false, retain=false 2024-11-13T18:13:12,245 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=5c8c2fb53a8f19068c501d6951c56b68, regionState=OPENING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:12,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c8c2fb53a8f19068c501d6951c56b68, ASSIGN because future has completed 2024-11-13T18:13:12,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:13:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T18:13:12,403 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:12,403 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:13:12,404 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,404 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:13:12,404 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,404 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,406 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,408 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:13:12,408 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c8c2fb53a8f19068c501d6951c56b68 columnFamilyName A 2024-11-13T18:13:12,408 DEBUG [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:12,409 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] regionserver.HStore(327): Store=5c8c2fb53a8f19068c501d6951c56b68/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:13:12,409 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,411 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:13:12,411 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c8c2fb53a8f19068c501d6951c56b68 columnFamilyName B 2024-11-13T18:13:12,411 DEBUG [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:12,411 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] regionserver.HStore(327): Store=5c8c2fb53a8f19068c501d6951c56b68/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:13:12,412 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,413 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:13:12,413 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c8c2fb53a8f19068c501d6951c56b68 columnFamilyName C 2024-11-13T18:13:12,413 DEBUG [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:12,413 INFO [StoreOpener-5c8c2fb53a8f19068c501d6951c56b68-1 {}] regionserver.HStore(327): Store=5c8c2fb53a8f19068c501d6951c56b68/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:13:12,413 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:12,413 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,414 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,414 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,415 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,415 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,416 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:13:12,417 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,418 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:13:12,419 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 5c8c2fb53a8f19068c501d6951c56b68; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73791079, jitterRate=0.09957276284694672}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:13:12,419 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:12,421 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 5c8c2fb53a8f19068c501d6951c56b68: Running coprocessor pre-open hook at 1731521592404Writing region info on filesystem at 1731521592405 (+1 ms)Initializing all the Stores at 1731521592406 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521592406Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521592406Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521592406Cleaning up temporary data from old regions at 1731521592415 (+9 ms)Running coprocessor post-open hooks at 1731521592419 (+4 ms)Region opened successfully at 1731521592421 (+2 ms) 2024-11-13T18:13:12,421 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., pid=53, masterSystemTime=1731521592399 2024-11-13T18:13:12,423 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:12,423 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:12,424 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=5c8c2fb53a8f19068c501d6951c56b68, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:12,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:13:12,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-13T18:13:12,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; OpenRegionProcedure 5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 in 179 msec 2024-11-13T18:13:12,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-13T18:13:12,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c8c2fb53a8f19068c501d6951c56b68, ASSIGN in 335 msec 2024-11-13T18:13:12,431 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:13:12,431 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521592431"}]},"ts":"1731521592431"} 2024-11-13T18:13:12,433 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:13:12,434 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:13:12,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-13T18:13:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T18:13:13,415 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:13,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b240986, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,417 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27306830, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4886d616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,420 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,423 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:13:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,37165,1731521525941 2024-11-13T18:13:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63377fe8 2024-11-13T18:13:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:13:13,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:13:13,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15faf580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,426 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37630aa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@186e3cd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4725022f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,430 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d4d256, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,431 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f231b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a4558c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,433 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cae2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,434 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc9c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5160eb1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,435 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79a8380b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,436 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c5abcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3425c658, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,438 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7088e370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,439 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63fa9d6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@627e095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,440 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67217dbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,442 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a86f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab147ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,443 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@717b2f2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,464 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@712f6aa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69cba14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,466 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aeb6b4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,467 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53fa97c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d102ea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,470 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383fc58f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,471 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa8555a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0e4e3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,472 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:13,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:13,472 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b70083a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:13,474 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd229bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:13,475 DEBUG [Thread-1196 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,475 DEBUG [Thread-1211 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,476 DEBUG [Thread-1194 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,477 DEBUG [Thread-1205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:13,477 DEBUG [Thread-1198 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,478 DEBUG [Thread-1192 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,479 DEBUG [Thread-1200 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-13T18:13:13,480 DEBUG [Thread-1209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:13:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:13:13,481 DEBUG [Thread-1203 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,481 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T18:13:13,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:13:13,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,483 DEBUG [Thread-1207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:13,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,484 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:13,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:13:13,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,486 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:13,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:13,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:13:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:13:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:13:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:13:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:13:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:13:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:13,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:13:13,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:13,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:13,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:13,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:13,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:13,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:13,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521653542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521593517, max timestamp=1731521593546Num uniques 859; 2024-11-13T18:13:13,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:13:13,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521593517, max timestamp=1731521593546Num uniques 859; 2024-11-13T18:13:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521593517, max timestamp=1731521593546Num uniques 859; 2024-11-13T18:13:13,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:13:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:13:13,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/c1fb31f207d243deb9b3d1e607bac5ed is 50, key is test_row_0/A:col10/1731521593490/Put/seqid=0 2024-11-13T18:13:13,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742084_1260 (size=12001) 2024-11-13T18:13:13,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/c1fb31f207d243deb9b3d1e607bac5ed 2024-11-13T18:13:13,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521653584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T18:13:13,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521653586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521653595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521653595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/5bc8a41f643e4248820611f5306c56e6 is 50, key is test_row_0/B:col10/1731521593490/Put/seqid=0 2024-11-13T18:13:13,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:13,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:13,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:13,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:13,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:13,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:13,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521653654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742085_1261 (size=12001) 2024-11-13T18:13:13,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/5bc8a41f643e4248820611f5306c56e6 2024-11-13T18:13:13,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/000750ab0c7142cc91275a81acb10f75 is 50, key is test_row_0/C:col10/1731521593490/Put/seqid=0 2024-11-13T18:13:13,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521653704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521653704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521653704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521653714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742086_1262 (size=12001) 2024-11-13T18:13:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T18:13:13,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:13,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:13,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:13,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:13,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:13,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:13,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521653864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521653914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521653915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521653915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521653924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:13,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:13,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:13,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:13,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T18:13:14,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/000750ab0c7142cc91275a81acb10f75 2024-11-13T18:13:14,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:14,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:14,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/c1fb31f207d243deb9b3d1e607bac5ed as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c1fb31f207d243deb9b3d1e607bac5ed 2024-11-13T18:13:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c1fb31f207d243deb9b3d1e607bac5ed, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:13:14,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/5bc8a41f643e4248820611f5306c56e6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/5bc8a41f643e4248820611f5306c56e6 2024-11-13T18:13:14,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/5bc8a41f643e4248820611f5306c56e6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:13:14,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/000750ab0c7142cc91275a81acb10f75 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/000750ab0c7142cc91275a81acb10f75 2024-11-13T18:13:14,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/000750ab0c7142cc91275a81acb10f75, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:13:14,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 5c8c2fb53a8f19068c501d6951c56b68 in 659ms, sequenceid=13, compaction requested=false 2024-11-13T18:13:14,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:14,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-13T18:13:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:14,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/a59178724dc3484bbb75833409fe571e is 50, key is test_row_0/A:col10/1731521593577/Put/seqid=0 2024-11-13T18:13:14,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742087_1263 (size=14341) 2024-11-13T18:13:14,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521654225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521654226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521654226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521654227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521654234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:14,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:14,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521654335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521654544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:14,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:14,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T18:13:14,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/a59178724dc3484bbb75833409fe571e 2024-11-13T18:13:14,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/e1f1e2616dd64760ac4dfcf1216ad4a1 is 50, key is test_row_0/B:col10/1731521593577/Put/seqid=0 2024-11-13T18:13:14,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742088_1264 (size=12001) 2024-11-13T18:13:14,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521654734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521654735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521654735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521654745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:14,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:14,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521654854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:14,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:14,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:14,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:14,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:15,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:15,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/e1f1e2616dd64760ac4dfcf1216ad4a1 2024-11-13T18:13:15,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/32e14c3043bd45d29a2795d269b7abeb is 50, key is test_row_0/C:col10/1731521593577/Put/seqid=0 2024-11-13T18:13:15,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742089_1265 (size=12001) 2024-11-13T18:13:15,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/32e14c3043bd45d29a2795d269b7abeb 2024-11-13T18:13:15,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:15,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:15,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/a59178724dc3484bbb75833409fe571e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/a59178724dc3484bbb75833409fe571e 2024-11-13T18:13:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/a59178724dc3484bbb75833409fe571e, entries=200, sequenceid=47, filesize=14.0 K 2024-11-13T18:13:15,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/e1f1e2616dd64760ac4dfcf1216ad4a1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e1f1e2616dd64760ac4dfcf1216ad4a1 2024-11-13T18:13:15,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e1f1e2616dd64760ac4dfcf1216ad4a1, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T18:13:15,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/32e14c3043bd45d29a2795d269b7abeb as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/32e14c3043bd45d29a2795d269b7abeb 2024-11-13T18:13:15,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/32e14c3043bd45d29a2795d269b7abeb, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T18:13:15,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 5c8c2fb53a8f19068c501d6951c56b68 in 1071ms, sequenceid=47, compaction requested=false 2024-11-13T18:13:15,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:15,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:13:15,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:15,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:15,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:15,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:15,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:15,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:15,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/0eaf1abaa6214e329f61d1856ba2d6df is 50, key is test_row_0/A:col10/1731521595367/Put/seqid=0 2024-11-13T18:13:15,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742090_1266 (size=14341) 2024-11-13T18:13:15,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/0eaf1abaa6214e329f61d1856ba2d6df 2024-11-13T18:13:15,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/a1fda2e3ba0f4ce59bbac189769a5b68 is 50, key is test_row_0/B:col10/1731521595367/Put/seqid=0 2024-11-13T18:13:15,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742091_1267 (size=12001) 2024-11-13T18:13:15,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/a1fda2e3ba0f4ce59bbac189769a5b68 2024-11-13T18:13:15,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:15,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:15,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d4b41d18d6e74f1689a090ca92036530 is 50, key is test_row_0/C:col10/1731521595367/Put/seqid=0 2024-11-13T18:13:15,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742092_1268 (size=12001) 2024-11-13T18:13:15,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d4b41d18d6e74f1689a090ca92036530 2024-11-13T18:13:15,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521655563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:15,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/0eaf1abaa6214e329f61d1856ba2d6df as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0eaf1abaa6214e329f61d1856ba2d6df 2024-11-13T18:13:15,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0eaf1abaa6214e329f61d1856ba2d6df, entries=200, sequenceid=59, filesize=14.0 K 2024-11-13T18:13:15,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/a1fda2e3ba0f4ce59bbac189769a5b68 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/a1fda2e3ba0f4ce59bbac189769a5b68 2024-11-13T18:13:15,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/a1fda2e3ba0f4ce59bbac189769a5b68, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T18:13:15,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d4b41d18d6e74f1689a090ca92036530 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d4b41d18d6e74f1689a090ca92036530 2024-11-13T18:13:15,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d4b41d18d6e74f1689a090ca92036530, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T18:13:15,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5c8c2fb53a8f19068c501d6951c56b68 in 246ms, sequenceid=59, compaction requested=true 2024-11-13T18:13:15,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:15,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:15,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:15,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:15,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:15,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:15,615 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:15,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:13:15,616 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:15,618 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:15,618 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:15,618 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,618 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/5bc8a41f643e4248820611f5306c56e6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e1f1e2616dd64760ac4dfcf1216ad4a1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/a1fda2e3ba0f4ce59bbac189769a5b68] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=35.2 K 2024-11-13T18:13:15,619 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:15,619 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:15,619 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,619 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c1fb31f207d243deb9b3d1e607bac5ed, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/a59178724dc3484bbb75833409fe571e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0eaf1abaa6214e329f61d1856ba2d6df] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=39.7 K 2024-11-13T18:13:15,621 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bc8a41f643e4248820611f5306c56e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731521593490 2024-11-13T18:13:15,623 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c1fb31f207d243deb9b3d1e607bac5ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731521593490 2024-11-13T18:13:15,623 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f1e2616dd64760ac4dfcf1216ad4a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731521593577 2024-11-13T18:13:15,623 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting a59178724dc3484bbb75833409fe571e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731521593577 2024-11-13T18:13:15,623 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1fda2e3ba0f4ce59bbac189769a5b68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521594205 2024-11-13T18:13:15,624 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eaf1abaa6214e329f61d1856ba2d6df, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521594191 2024-11-13T18:13:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T18:13:15,643 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:15,644 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/00206d648b594de59098cd1c8b51fdf3 is 50, key is test_row_0/B:col10/1731521595367/Put/seqid=0 2024-11-13T18:13:15,655 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#220 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:15,655 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/6403e30f04804dcab0640979cafd0a8d is 50, key is test_row_0/A:col10/1731521595367/Put/seqid=0 2024-11-13T18:13:15,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742093_1269 (size=12104) 2024-11-13T18:13:15,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521595456, max timestamp=1731521595690Num uniques 859; 2024-11-13T18:13:15,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:15,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521595456, max timestamp=1731521595690Num uniques 859; 2024-11-13T18:13:15,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:15,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-13T18:13:15,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:15,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:15,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:15,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:15,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:15,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:15,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742094_1270 (size=12104) 2024-11-13T18:13:15,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/98f2a2cb4a5647bba78c444915c9a4c2 is 50, key is test_row_0/A:col10/1731521595559/Put/seqid=0 2024-11-13T18:13:15,732 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/6403e30f04804dcab0640979cafd0a8d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6403e30f04804dcab0640979cafd0a8d 2024-11-13T18:13:15,741 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into 6403e30f04804dcab0640979cafd0a8d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:15,741 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:15,741 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=13, startTime=1731521595615; duration=0sec 2024-11-13T18:13:15,741 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:15,741 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:15,741 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:15,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742095_1271 (size=12001) 2024-11-13T18:13:15,743 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:15,743 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:15,743 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,743 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/000750ab0c7142cc91275a81acb10f75, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/32e14c3043bd45d29a2795d269b7abeb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d4b41d18d6e74f1689a090ca92036530] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=35.2 K 2024-11-13T18:13:15,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/98f2a2cb4a5647bba78c444915c9a4c2 2024-11-13T18:13:15,744 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 000750ab0c7142cc91275a81acb10f75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731521593490 2024-11-13T18:13:15,745 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 32e14c3043bd45d29a2795d269b7abeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731521593577 2024-11-13T18:13:15,745 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b41d18d6e74f1689a090ca92036530, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521594205 2024-11-13T18:13:15,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f696c3283a3c4a6d8e696397e265331b is 50, key is test_row_0/B:col10/1731521595559/Put/seqid=0 2024-11-13T18:13:15,774 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:15,774 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/8b799188969c4fc5986d0b99099c13eb is 50, key is test_row_0/C:col10/1731521595367/Put/seqid=0 2024-11-13T18:13:15,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521595785, max timestamp=1731521595791Num uniques 859; 2024-11-13T18:13:15,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521595785, max timestamp=1731521595791Num uniques 859; 2024-11-13T18:13:15,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:15,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:15,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521595785, max timestamp=1731521595791Num uniques 859; 2024-11-13T18:13:15,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:15,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742096_1272 (size=12001) 2024-11-13T18:13:15,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f696c3283a3c4a6d8e696397e265331b 2024-11-13T18:13:15,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742097_1273 (size=12104) 2024-11-13T18:13:15,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521655818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:15,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:15,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521655821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:15,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:15,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/da7e0e16478847f483e70f6496667059 is 50, key is test_row_0/C:col10/1731521595559/Put/seqid=0 2024-11-13T18:13:15,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521655825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:15,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521655828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:15,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521655829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:15,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:15,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:15,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:15,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:15,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742098_1274 (size=12001) 2024-11-13T18:13:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:15,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/da7e0e16478847f483e70f6496667059 2024-11-13T18:13:15,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/98f2a2cb4a5647bba78c444915c9a4c2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/98f2a2cb4a5647bba78c444915c9a4c2 2024-11-13T18:13:15,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/98f2a2cb4a5647bba78c444915c9a4c2, entries=150, sequenceid=80, filesize=11.7 K 2024-11-13T18:13:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f696c3283a3c4a6d8e696397e265331b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f696c3283a3c4a6d8e696397e265331b 2024-11-13T18:13:15,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f696c3283a3c4a6d8e696397e265331b, entries=150, sequenceid=80, filesize=11.7 K 2024-11-13T18:13:15,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/da7e0e16478847f483e70f6496667059 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da7e0e16478847f483e70f6496667059 2024-11-13T18:13:15,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da7e0e16478847f483e70f6496667059, entries=150, sequenceid=80, filesize=11.7 K 2024-11-13T18:13:15,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 5c8c2fb53a8f19068c501d6951c56b68 in 196ms, sequenceid=80, compaction requested=false 2024-11-13T18:13:15,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:15,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=134.18 KB heapSize=385.22 KB 2024-11-13T18:13:15,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521595824, max timestamp=1731521595941Num uniques 1018; 2024-11-13T18:13:15,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:13:15,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521595824, max timestamp=1731521595941Num uniques 1018; 2024-11-13T18:13:15,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521595824, max timestamp=1731521595941Num uniques 1018; 2024-11-13T18:13:15,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:13:15,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:13:15,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:15,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:15,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:15,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:15,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:15,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:15,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/281ea65017754a929bb84acf8c30996d is 50, key is test_row_0/A:col10/1731521595826/Put/seqid=0 2024-11-13T18:13:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742099_1275 (size=12001) 2024-11-13T18:13:15,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/281ea65017754a929bb84acf8c30996d 2024-11-13T18:13:16,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:16,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:16,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:16,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:16,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:16,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:16,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521595990, max timestamp=1731521596005Num uniques 859; 2024-11-13T18:13:16,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:16,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521595990, max timestamp=1731521596005Num uniques 859; 2024-11-13T18:13:16,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:16,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521595990, max timestamp=1731521596005Num uniques 859; 2024-11-13T18:13:16,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:16,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/794ab2da1e7d41aab3863992b599ced2 is 50, key is test_row_0/B:col10/1731521595826/Put/seqid=0 2024-11-13T18:13:16,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742100_1276 (size=12001) 2024-11-13T18:13:16,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/794ab2da1e7d41aab3863992b599ced2 2024-11-13T18:13:16,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731521596005, max timestamp=1731521596071Num uniques 1018; 2024-11-13T18:13:16,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T18:13:16,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521596005, max timestamp=1731521596071Num uniques 1018; 2024-11-13T18:13:16,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:13:16,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521596005, max timestamp=1731521596075Num uniques 1018; 2024-11-13T18:13:16,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:13:16,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d1722c2971834959a7c8b794ff032da8 is 50, key is test_row_0/C:col10/1731521595826/Put/seqid=0 2024-11-13T18:13:16,090 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/00206d648b594de59098cd1c8b51fdf3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/00206d648b594de59098cd1c8b51fdf3 2024-11-13T18:13:16,100 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into 00206d648b594de59098cd1c8b51fdf3(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:16,100 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:16,100 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=13, startTime=1731521595615; duration=0sec 2024-11-13T18:13:16,100 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:16,100 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:16,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742101_1277 (size=12001) 2024-11-13T18:13:16,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d1722c2971834959a7c8b794ff032da8 2024-11-13T18:13:16,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731521596115, max timestamp=1731521596128Num uniques 1077; 2024-11-13T18:13:16,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-13T18:13:16,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/281ea65017754a929bb84acf8c30996d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/281ea65017754a929bb84acf8c30996d 2024-11-13T18:13:16,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731521596115, max timestamp=1731521596128Num uniques 1027; 2024-11-13T18:13:16,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T18:13:16,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731521596115, max timestamp=1731521596128Num uniques 1027; 2024-11-13T18:13:16,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T18:13:16,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/281ea65017754a929bb84acf8c30996d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T18:13:16,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/794ab2da1e7d41aab3863992b599ced2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/794ab2da1e7d41aab3863992b599ced2 2024-11-13T18:13:16,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/794ab2da1e7d41aab3863992b599ced2, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T18:13:16,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d1722c2971834959a7c8b794ff032da8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d1722c2971834959a7c8b794ff032da8 2024-11-13T18:13:16,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d1722c2971834959a7c8b794ff032da8, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T18:13:16,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:16,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:16,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:16,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:16,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:16,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:16,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=89.45 KB/91600 for 5c8c2fb53a8f19068c501d6951c56b68 in 228ms, sequenceid=119, compaction requested=true 2024-11-13T18:13:16,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:16,170 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:16,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:16,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:16,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:16,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:16,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:16,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:13:16,171 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:16,171 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:16,171 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:16,172 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6403e30f04804dcab0640979cafd0a8d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/98f2a2cb4a5647bba78c444915c9a4c2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/281ea65017754a929bb84acf8c30996d] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=35.3 K 2024-11-13T18:13:16,172 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6403e30f04804dcab0640979cafd0a8d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521594205 2024-11-13T18:13:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:16,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=96.16 KB heapSize=268.84 KB 2024-11-13T18:13:16,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:16,173 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98f2a2cb4a5647bba78c444915c9a4c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731521595516 2024-11-13T18:13:16,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:16,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:16,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:16,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:16,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:16,174 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 281ea65017754a929bb84acf8c30996d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731521595824 2024-11-13T18:13:16,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/650c434ffc12479fbe1f5f1d50611b21 is 50, key is test_row_0/A:col10/1731521596166/Put/seqid=0 2024-11-13T18:13:16,195 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:16,195 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/1839a745aa4949d6b26e1be5be2206f1 is 50, key is test_row_0/A:col10/1731521595826/Put/seqid=0 2024-11-13T18:13:16,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742103_1279 (size=12207) 2024-11-13T18:13:16,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742102_1278 (size=14541) 2024-11-13T18:13:16,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/650c434ffc12479fbe1f5f1d50611b21 2024-11-13T18:13:16,235 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/1839a745aa4949d6b26e1be5be2206f1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1839a745aa4949d6b26e1be5be2206f1 2024-11-13T18:13:16,242 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/8b799188969c4fc5986d0b99099c13eb as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8b799188969c4fc5986d0b99099c13eb 2024-11-13T18:13:16,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/e51006c168634843b91d1b0586407136 is 50, key is test_row_0/B:col10/1731521596166/Put/seqid=0 2024-11-13T18:13:16,264 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into 1839a745aa4949d6b26e1be5be2206f1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:16,264 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:16,264 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=13, startTime=1731521596170; duration=0sec 2024-11-13T18:13:16,265 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:13:16,265 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:16,265 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-13T18:13:16,266 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:13:16,266 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:13:16,266 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. because compaction request was cancelled 2024-11-13T18:13:16,266 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:16,266 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:16,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521656205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,267 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into 8b799188969c4fc5986d0b99099c13eb(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:16,267 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:16,267 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=13, startTime=1731521595615; duration=0sec 2024-11-13T18:13:16,267 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:16,267 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,268 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,268 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,268 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:16,268 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/00206d648b594de59098cd1c8b51fdf3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f696c3283a3c4a6d8e696397e265331b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/794ab2da1e7d41aab3863992b599ced2] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=35.3 K 2024-11-13T18:13:16,269 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00206d648b594de59098cd1c8b51fdf3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521594205 2024-11-13T18:13:16,270 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f696c3283a3c4a6d8e696397e265331b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731521595516 2024-11-13T18:13:16,271 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 794ab2da1e7d41aab3863992b599ced2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731521595824 2024-11-13T18:13:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521656267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521656267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521656268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521656270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,310 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:16,311 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/59ce033e73cc4c6491840fdc26ad0961 is 50, key is test_row_0/B:col10/1731521595826/Put/seqid=0 2024-11-13T18:13:16,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742104_1280 (size=12151) 2024-11-13T18:13:16,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/e51006c168634843b91d1b0586407136 2024-11-13T18:13:16,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:16,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:16,330 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37165 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fda96cf025dc,44483,1731521526892, table=TestAcidGuarantees, region=5c8c2fb53a8f19068c501d6951c56b68. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:13:16,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a5f57fec9b5d497d9ea0912c4d8f5802 is 50, key is test_row_0/C:col10/1731521596166/Put/seqid=0 2024-11-13T18:13:16,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742105_1281 (size=12207) 2024-11-13T18:13:16,380 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/59ce033e73cc4c6491840fdc26ad0961 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/59ce033e73cc4c6491840fdc26ad0961 2024-11-13T18:13:16,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521656374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,387 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into 59ce033e73cc4c6491840fdc26ad0961(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:16,387 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:16,387 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=13, startTime=1731521596171; duration=0sec 2024-11-13T18:13:16,387 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:16,387 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:16,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742106_1282 (size=12151) 2024-11-13T18:13:16,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a5f57fec9b5d497d9ea0912c4d8f5802 2024-11-13T18:13:16,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/650c434ffc12479fbe1f5f1d50611b21 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/650c434ffc12479fbe1f5f1d50611b21 2024-11-13T18:13:16,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521656394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521656395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/650c434ffc12479fbe1f5f1d50611b21, entries=200, sequenceid=185, filesize=14.2 K 2024-11-13T18:13:16,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521656395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521656395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/e51006c168634843b91d1b0586407136 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e51006c168634843b91d1b0586407136 2024-11-13T18:13:16,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e51006c168634843b91d1b0586407136, entries=150, sequenceid=185, filesize=11.9 K 2024-11-13T18:13:16,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a5f57fec9b5d497d9ea0912c4d8f5802 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5f57fec9b5d497d9ea0912c4d8f5802 2024-11-13T18:13:16,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5f57fec9b5d497d9ea0912c4d8f5802, entries=150, sequenceid=185, filesize=11.9 K 2024-11-13T18:13:16,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~268.80 KB/275248, currentSize=93.93 KB/96180 for 5c8c2fb53a8f19068c501d6951c56b68 in 242ms, sequenceid=185, compaction requested=true 2024-11-13T18:13:16,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:16,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:16,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:16,416 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:13:16,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:16,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:16,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:16,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:16,416 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:13:16,417 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:13:16,417 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:13:16,417 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. because compaction request was cancelled 2024-11-13T18:13:16,417 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:13:16,417 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:16,417 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:13:16,417 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:13:16,417 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. because compaction request was cancelled 2024-11-13T18:13:16,417 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:16,418 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:13:16,418 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:16,418 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:16,419 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8b799188969c4fc5986d0b99099c13eb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da7e0e16478847f483e70f6496667059, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d1722c2971834959a7c8b794ff032da8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5f57fec9b5d497d9ea0912c4d8f5802] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=47.1 K 2024-11-13T18:13:16,420 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b799188969c4fc5986d0b99099c13eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521594205 2024-11-13T18:13:16,420 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7e0e16478847f483e70f6496667059, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731521595516 2024-11-13T18:13:16,420 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1722c2971834959a7c8b794ff032da8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731521595824 2024-11-13T18:13:16,421 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5f57fec9b5d497d9ea0912c4d8f5802, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731521596151 2024-11-13T18:13:16,441 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#233 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:16,444 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/6781a7b94e5940d3872ea2bb977a1644 is 50, key is test_row_0/C:col10/1731521596166/Put/seqid=0 2024-11-13T18:13:16,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T18:13:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:16,481 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:13:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:16,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742107_1283 (size=12391) 2024-11-13T18:13:16,499 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/6781a7b94e5940d3872ea2bb977a1644 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/6781a7b94e5940d3872ea2bb977a1644 2024-11-13T18:13:16,506 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into 6781a7b94e5940d3872ea2bb977a1644(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:16,506 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:16,506 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=12, startTime=1731521596416; duration=0sec 2024-11-13T18:13:16,506 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:16,506 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/1225a14fd7424c018403dafc90209bbe is 50, key is test_row_0/A:col10/1731521596216/Put/seqid=0 2024-11-13T18:13:16,540 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:13:16,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742108_1284 (size=12151) 2024-11-13T18:13:16,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:16,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521656619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521656621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521656623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521656627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521656627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521656734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521656735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521656735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521656747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521656747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521656944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521656944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521656944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,955 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/1225a14fd7424c018403dafc90209bbe 2024-11-13T18:13:16,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521656954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521656954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/52cda14cffe644b3a00b562f29f4b6c7 is 50, key is test_row_0/B:col10/1731521596216/Put/seqid=0 2024-11-13T18:13:16,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742109_1285 (size=12151) 2024-11-13T18:13:16,991 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/52cda14cffe644b3a00b562f29f4b6c7 2024-11-13T18:13:17,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d58b3f90d56145a58d6457656896dfe7 is 50, key is test_row_0/C:col10/1731521596216/Put/seqid=0 2024-11-13T18:13:17,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742110_1286 (size=12151) 2024-11-13T18:13:17,016 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d58b3f90d56145a58d6457656896dfe7 2024-11-13T18:13:17,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/1225a14fd7424c018403dafc90209bbe as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1225a14fd7424c018403dafc90209bbe 2024-11-13T18:13:17,027 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1225a14fd7424c018403dafc90209bbe, entries=150, sequenceid=205, filesize=11.9 K 2024-11-13T18:13:17,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/52cda14cffe644b3a00b562f29f4b6c7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/52cda14cffe644b3a00b562f29f4b6c7 2024-11-13T18:13:17,032 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/52cda14cffe644b3a00b562f29f4b6c7, entries=150, sequenceid=205, filesize=11.9 K 2024-11-13T18:13:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d58b3f90d56145a58d6457656896dfe7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d58b3f90d56145a58d6457656896dfe7 2024-11-13T18:13:17,038 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d58b3f90d56145a58d6457656896dfe7, entries=150, sequenceid=205, filesize=11.9 K 2024-11-13T18:13:17,039 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 5c8c2fb53a8f19068c501d6951c56b68 in 558ms, sequenceid=205, compaction requested=true 2024-11-13T18:13:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-13T18:13:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-13T18:13:17,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-13T18:13:17,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5540 sec 2024-11-13T18:13:17,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 3.5660 sec 2024-11-13T18:13:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:17,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:13:17,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:17,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:17,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:17,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:17,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:17,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:17,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/e4279a1164304a13a64a5811c872ff02 is 50, key is test_row_0/A:col10/1731521596621/Put/seqid=0 2024-11-13T18:13:17,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521657283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521657283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521657288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521657294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521657297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742111_1287 (size=12151) 2024-11-13T18:13:17,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/e4279a1164304a13a64a5811c872ff02 2024-11-13T18:13:17,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/c9cd7cf411bf418b93f53f8ac1600d92 is 50, key is test_row_0/B:col10/1731521596621/Put/seqid=0 2024-11-13T18:13:17,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742112_1288 (size=12151) 2024-11-13T18:13:17,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/c9cd7cf411bf418b93f53f8ac1600d92 2024-11-13T18:13:17,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/896c6704f83e4a3fad39d9e131e7f9ed is 50, key is test_row_0/C:col10/1731521596621/Put/seqid=0 2024-11-13T18:13:17,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742113_1289 (size=12151) 2024-11-13T18:13:17,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521657404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521657404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521657404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521657404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521657404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521657614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521657614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521657614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521657618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521657618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T18:13:17,654 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:17,655 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-13T18:13:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:13:17,658 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:17,660 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:17,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:13:17,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/896c6704f83e4a3fad39d9e131e7f9ed 2024-11-13T18:13:17,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/e4279a1164304a13a64a5811c872ff02 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e4279a1164304a13a64a5811c872ff02 2024-11-13T18:13:17,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e4279a1164304a13a64a5811c872ff02, entries=150, sequenceid=224, filesize=11.9 K 2024-11-13T18:13:17,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/c9cd7cf411bf418b93f53f8ac1600d92 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/c9cd7cf411bf418b93f53f8ac1600d92 2024-11-13T18:13:17,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/c9cd7cf411bf418b93f53f8ac1600d92, entries=150, sequenceid=224, filesize=11.9 K 2024-11-13T18:13:17,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/896c6704f83e4a3fad39d9e131e7f9ed as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/896c6704f83e4a3fad39d9e131e7f9ed 2024-11-13T18:13:17,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T18:13:17,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:17,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:17,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:17,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:17,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:17,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/896c6704f83e4a3fad39d9e131e7f9ed, entries=150, sequenceid=224, filesize=11.9 K 2024-11-13T18:13:17,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 5c8c2fb53a8f19068c501d6951c56b68 in 559ms, sequenceid=224, compaction requested=true 2024-11-13T18:13:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:17,819 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:13:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:17,819 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:13:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:17,820 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:13:17,820 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:13:17,820 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:17,820 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:17,820 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:17,821 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:17,821 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/59ce033e73cc4c6491840fdc26ad0961, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e51006c168634843b91d1b0586407136, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/52cda14cffe644b3a00b562f29f4b6c7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/c9cd7cf411bf418b93f53f8ac1600d92] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=47.5 K 2024-11-13T18:13:17,821 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1839a745aa4949d6b26e1be5be2206f1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/650c434ffc12479fbe1f5f1d50611b21, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1225a14fd7424c018403dafc90209bbe, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e4279a1164304a13a64a5811c872ff02] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=49.9 K 2024-11-13T18:13:17,822 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 59ce033e73cc4c6491840fdc26ad0961, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731521595824 2024-11-13T18:13:17,822 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1839a745aa4949d6b26e1be5be2206f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731521595824 2024-11-13T18:13:17,822 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e51006c168634843b91d1b0586407136, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731521596151 2024-11-13T18:13:17,822 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 650c434ffc12479fbe1f5f1d50611b21, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731521596151 2024-11-13T18:13:17,823 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1225a14fd7424c018403dafc90209bbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731521596205 2024-11-13T18:13:17,823 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 52cda14cffe644b3a00b562f29f4b6c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731521596205 2024-11-13T18:13:17,823 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c9cd7cf411bf418b93f53f8ac1600d92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521596621 2024-11-13T18:13:17,823 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4279a1164304a13a64a5811c872ff02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521596621 2024-11-13T18:13:17,836 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:17,836 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/dba5d51785854eada9cf4c4c8eadda07 is 50, key is test_row_0/B:col10/1731521596621/Put/seqid=0 2024-11-13T18:13:17,851 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:17,852 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/99f46b5783f84d229f7ade93b00a8ff3 is 50, key is test_row_0/A:col10/1731521596621/Put/seqid=0 2024-11-13T18:13:17,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742114_1290 (size=12493) 2024-11-13T18:13:17,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742115_1291 (size=12493) 2024-11-13T18:13:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:17,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:13:17,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:17,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:17,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:17,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:17,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:17,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:17,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/fdf3dd2d10c541ecb69ce5a27cd4d288 is 50, key is test_row_0/A:col10/1731521597926/Put/seqid=0 2024-11-13T18:13:17,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742116_1292 (size=14541) 2024-11-13T18:13:17,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521657943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521657943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521657945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521657946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521657947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:17,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:17,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:17,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T18:13:17,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:17,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:17,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:17,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:17,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:13:18,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521658054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521658055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521658055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521658064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521658064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T18:13:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:18,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,265 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/dba5d51785854eada9cf4c4c8eadda07 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/dba5d51785854eada9cf4c4c8eadda07 2024-11-13T18:13:18,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521658265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521658265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,270 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/99f46b5783f84d229f7ade93b00a8ff3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/99f46b5783f84d229f7ade93b00a8ff3 2024-11-13T18:13:18,273 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into dba5d51785854eada9cf4c4c8eadda07(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:18,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521658272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,274 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:18,274 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=12, startTime=1731521597819; duration=0sec 2024-11-13T18:13:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,274 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:18,274 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:18,274 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:18,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T18:13:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:18,275 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into 99f46b5783f84d229f7ade93b00a8ff3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,275 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:18,275 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=12, startTime=1731521597819; duration=0sec 2024-11-13T18:13:18,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,275 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:18,275 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,275 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:18,276 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:18,276 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,276 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/6781a7b94e5940d3872ea2bb977a1644, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d58b3f90d56145a58d6457656896dfe7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/896c6704f83e4a3fad39d9e131e7f9ed] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=35.8 K 2024-11-13T18:13:18,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521658274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,276 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6781a7b94e5940d3872ea2bb977a1644, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731521596151 2024-11-13T18:13:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521658274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,278 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d58b3f90d56145a58d6457656896dfe7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731521596205 2024-11-13T18:13:18,279 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 896c6704f83e4a3fad39d9e131e7f9ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521596621 2024-11-13T18:13:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:13:18,297 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:18,297 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/c69b950d057541e2af6854b190b4bb8a is 50, key is test_row_0/C:col10/1731521596621/Put/seqid=0 2024-11-13T18:13:18,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742117_1293 (size=12493) 2024-11-13T18:13:18,315 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/c69b950d057541e2af6854b190b4bb8a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/c69b950d057541e2af6854b190b4bb8a 2024-11-13T18:13:18,329 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into c69b950d057541e2af6854b190b4bb8a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:18,329 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:18,329 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=13, startTime=1731521597819; duration=0sec 2024-11-13T18:13:18,329 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:18,329 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:18,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/fdf3dd2d10c541ecb69ce5a27cd4d288 2024-11-13T18:13:18,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/7826e36e52744ffeb3f62f114c85e100 is 50, key is test_row_0/B:col10/1731521597926/Put/seqid=0 2024-11-13T18:13:18,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742118_1294 (size=12151) 2024-11-13T18:13:18,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T18:13:18,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:18,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521658574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521658574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T18:13:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521658584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521658584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521658584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:18,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T18:13:18,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:18,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:18,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/7826e36e52744ffeb3f62f114c85e100 2024-11-13T18:13:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:13:18,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b9e6789dce0146938b94f031cf31f32b is 50, key is test_row_0/C:col10/1731521597926/Put/seqid=0 2024-11-13T18:13:18,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742119_1295 (size=12151) 2024-11-13T18:13:18,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b9e6789dce0146938b94f031cf31f32b 2024-11-13T18:13:18,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/fdf3dd2d10c541ecb69ce5a27cd4d288 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/fdf3dd2d10c541ecb69ce5a27cd4d288 2024-11-13T18:13:18,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/fdf3dd2d10c541ecb69ce5a27cd4d288, entries=200, sequenceid=241, filesize=14.2 K 2024-11-13T18:13:18,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/7826e36e52744ffeb3f62f114c85e100 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7826e36e52744ffeb3f62f114c85e100 2024-11-13T18:13:18,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7826e36e52744ffeb3f62f114c85e100, entries=150, sequenceid=241, filesize=11.9 K 2024-11-13T18:13:18,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b9e6789dce0146938b94f031cf31f32b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b9e6789dce0146938b94f031cf31f32b 2024-11-13T18:13:18,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b9e6789dce0146938b94f031cf31f32b, entries=150, sequenceid=241, filesize=11.9 K 2024-11-13T18:13:18,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5c8c2fb53a8f19068c501d6951c56b68 in 917ms, sequenceid=241, compaction requested=false 2024-11-13T18:13:18,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:18,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T18:13:18,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:18,892 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:13:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:18,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/f5690cdaa7224797b14e4b6520bf3ec7 is 50, key is test_row_0/A:col10/1731521597945/Put/seqid=0 2024-11-13T18:13:18,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742120_1296 (size=12151) 2024-11-13T18:13:18,902 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/f5690cdaa7224797b14e4b6520bf3ec7 2024-11-13T18:13:18,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/1f60a5f0fff14d40b14a6a94c8ddc153 is 50, key is test_row_0/B:col10/1731521597945/Put/seqid=0 2024-11-13T18:13:18,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742121_1297 (size=12151) 2024-11-13T18:13:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:19,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:19,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521659114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521659115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521659117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521659117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521659121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521659224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521659224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521659224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521659234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521659234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,333 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/1f60a5f0fff14d40b14a6a94c8ddc153 2024-11-13T18:13:19,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b61815199c014bd5951ae2e6843106ec is 50, key is test_row_0/C:col10/1731521597945/Put/seqid=0 2024-11-13T18:13:19,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742122_1298 (size=12151) 2024-11-13T18:13:19,357 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b61815199c014bd5951ae2e6843106ec 2024-11-13T18:13:19,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/f5690cdaa7224797b14e4b6520bf3ec7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f5690cdaa7224797b14e4b6520bf3ec7 2024-11-13T18:13:19,367 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f5690cdaa7224797b14e4b6520bf3ec7, entries=150, sequenceid=259, filesize=11.9 K 2024-11-13T18:13:19,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/1f60a5f0fff14d40b14a6a94c8ddc153 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/1f60a5f0fff14d40b14a6a94c8ddc153 2024-11-13T18:13:19,372 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/1f60a5f0fff14d40b14a6a94c8ddc153, entries=150, sequenceid=259, filesize=11.9 K 2024-11-13T18:13:19,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b61815199c014bd5951ae2e6843106ec as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b61815199c014bd5951ae2e6843106ec 2024-11-13T18:13:19,385 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b61815199c014bd5951ae2e6843106ec, entries=150, sequenceid=259, filesize=11.9 K 2024-11-13T18:13:19,387 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5c8c2fb53a8f19068c501d6951c56b68 in 495ms, sequenceid=259, compaction requested=true 2024-11-13T18:13:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-13T18:13:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-13T18:13:19,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-13T18:13:19,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7330 sec 2024-11-13T18:13:19,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.7410 sec 2024-11-13T18:13:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:19,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:13:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:19,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/48530d3573774907a619aafde6ce6a7c is 50, key is test_row_0/A:col10/1731521599120/Put/seqid=0 2024-11-13T18:13:19,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742123_1299 (size=12301) 2024-11-13T18:13:19,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/48530d3573774907a619aafde6ce6a7c 2024-11-13T18:13:19,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521659453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521659455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521659456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521659458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521659460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/89db2d89a9734ab6a490f9eacc377ed7 is 50, key is test_row_0/B:col10/1731521599120/Put/seqid=0 2024-11-13T18:13:19,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742124_1300 (size=12301) 2024-11-13T18:13:19,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521659565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521659565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521659565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521659574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521659574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521659774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521659774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521659775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521659784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521659784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:13:19,804 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:19,805 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-13T18:13:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:13:19,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:19,808 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:19,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:19,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/89db2d89a9734ab6a490f9eacc377ed7 2024-11-13T18:13:19,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b7976af3804e456ea7ddd42d646f63c9 is 50, key is test_row_0/C:col10/1731521599120/Put/seqid=0 2024-11-13T18:13:19,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742125_1301 (size=12301) 2024-11-13T18:13:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:13:19,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:13:19,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:19,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:19,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:19,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:19,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:20,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521660084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521660084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521660084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521660095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521660095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:13:20,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:20,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:20,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:20,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:20,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:13:20,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:13:20,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:20,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:20,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:20,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:20,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:20,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b7976af3804e456ea7ddd42d646f63c9 2024-11-13T18:13:20,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/48530d3573774907a619aafde6ce6a7c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/48530d3573774907a619aafde6ce6a7c 2024-11-13T18:13:20,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/48530d3573774907a619aafde6ce6a7c, entries=150, sequenceid=278, filesize=12.0 K 2024-11-13T18:13:20,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/89db2d89a9734ab6a490f9eacc377ed7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/89db2d89a9734ab6a490f9eacc377ed7 2024-11-13T18:13:20,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/89db2d89a9734ab6a490f9eacc377ed7, entries=150, sequenceid=278, filesize=12.0 K 2024-11-13T18:13:20,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b7976af3804e456ea7ddd42d646f63c9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b7976af3804e456ea7ddd42d646f63c9 2024-11-13T18:13:20,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b7976af3804e456ea7ddd42d646f63c9, entries=150, sequenceid=278, filesize=12.0 K 2024-11-13T18:13:20,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5c8c2fb53a8f19068c501d6951c56b68 in 937ms, sequenceid=278, compaction requested=true 2024-11-13T18:13:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:20,375 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:13:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:13:20,376 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:13:20,377 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51486 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:13:20,377 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:20,377 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:20,378 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/99f46b5783f84d229f7ade93b00a8ff3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/fdf3dd2d10c541ecb69ce5a27cd4d288, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f5690cdaa7224797b14e4b6520bf3ec7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/48530d3573774907a619aafde6ce6a7c] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=50.3 K 2024-11-13T18:13:20,378 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 99f46b5783f84d229f7ade93b00a8ff3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521596621 2024-11-13T18:13:20,385 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting fdf3dd2d10c541ecb69ce5a27cd4d288, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731521597280 2024-11-13T18:13:20,388 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f5690cdaa7224797b14e4b6520bf3ec7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731521597941 2024-11-13T18:13:20,388 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:13:20,388 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 48530d3573774907a619aafde6ce6a7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731521599120 2024-11-13T18:13:20,388 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:20,388 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:20,388 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/dba5d51785854eada9cf4c4c8eadda07, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7826e36e52744ffeb3f62f114c85e100, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/1f60a5f0fff14d40b14a6a94c8ddc153, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/89db2d89a9734ab6a490f9eacc377ed7] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=47.9 K 2024-11-13T18:13:20,389 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting dba5d51785854eada9cf4c4c8eadda07, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521596621 2024-11-13T18:13:20,393 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7826e36e52744ffeb3f62f114c85e100, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731521597287 2024-11-13T18:13:20,393 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f60a5f0fff14d40b14a6a94c8ddc153, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731521597941 2024-11-13T18:13:20,397 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89db2d89a9734ab6a490f9eacc377ed7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731521599120 2024-11-13T18:13:20,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:13:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:13:20,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:20,437 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:13:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:20,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/d095cf21b421409ab1b808fd8798d1cf is 50, key is test_row_0/A:col10/1731521599457/Put/seqid=0 2024-11-13T18:13:20,458 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:20,459 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6fd7bc6a81304c7f8327f5e6a14ca782 is 50, key is test_row_0/B:col10/1731521599120/Put/seqid=0 2024-11-13T18:13:20,477 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#254 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:20,477 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/89e26d57f51b42e298a828974112437e is 50, key is test_row_0/A:col10/1731521599120/Put/seqid=0 2024-11-13T18:13:20,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742126_1302 (size=12779) 2024-11-13T18:13:20,533 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6fd7bc6a81304c7f8327f5e6a14ca782 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6fd7bc6a81304c7f8327f5e6a14ca782 2024-11-13T18:13:20,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742127_1303 (size=12301) 2024-11-13T18:13:20,541 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into 6fd7bc6a81304c7f8327f5e6a14ca782(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:20,541 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:20,541 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=12, startTime=1731521600375; duration=0sec 2024-11-13T18:13:20,541 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:20,541 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:20,541 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:13:20,542 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:13:20,542 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:20,542 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:20,543 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/c69b950d057541e2af6854b190b4bb8a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b9e6789dce0146938b94f031cf31f32b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b61815199c014bd5951ae2e6843106ec, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b7976af3804e456ea7ddd42d646f63c9] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=47.9 K 2024-11-13T18:13:20,543 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c69b950d057541e2af6854b190b4bb8a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521596621 2024-11-13T18:13:20,545 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e6789dce0146938b94f031cf31f32b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731521597287 2024-11-13T18:13:20,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742128_1304 (size=12779) 2024-11-13T18:13:20,546 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b61815199c014bd5951ae2e6843106ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731521597941 2024-11-13T18:13:20,552 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7976af3804e456ea7ddd42d646f63c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731521599120 2024-11-13T18:13:20,559 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/89e26d57f51b42e298a828974112437e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/89e26d57f51b42e298a828974112437e 2024-11-13T18:13:20,570 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into 89e26d57f51b42e298a828974112437e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:20,570 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:20,570 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=12, startTime=1731521600375; duration=0sec 2024-11-13T18:13:20,570 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:20,570 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:20,575 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:20,576 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/0133a20649f54e1587938ac639392758 is 50, key is test_row_0/C:col10/1731521599120/Put/seqid=0 2024-11-13T18:13:20,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742129_1305 (size=12779) 2024-11-13T18:13:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:20,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:20,648 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/0133a20649f54e1587938ac639392758 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/0133a20649f54e1587938ac639392758 2024-11-13T18:13:20,661 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into 0133a20649f54e1587938ac639392758(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:20,661 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:20,661 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=12, startTime=1731521600375; duration=0sec 2024-11-13T18:13:20,661 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:20,661 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:20,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521660662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521660664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521660664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521660665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521660667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521660774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521660784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521660784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521660784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521660784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:13:20,944 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/d095cf21b421409ab1b808fd8798d1cf 2024-11-13T18:13:20,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f5850f3eb01a489080812834455ccea2 is 50, key is test_row_0/B:col10/1731521599457/Put/seqid=0 2024-11-13T18:13:20,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521660984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742130_1306 (size=12301) 2024-11-13T18:13:20,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521660994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521660994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521660994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:20,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521660995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521661294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521661304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521661304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521661304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521661305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,393 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f5850f3eb01a489080812834455ccea2 2024-11-13T18:13:21,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/da943ba3e98c424699a33a46669e3a61 is 50, key is test_row_0/C:col10/1731521599457/Put/seqid=0 2024-11-13T18:13:21,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742131_1307 (size=12301) 2024-11-13T18:13:21,442 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/da943ba3e98c424699a33a46669e3a61 2024-11-13T18:13:21,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/d095cf21b421409ab1b808fd8798d1cf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d095cf21b421409ab1b808fd8798d1cf 2024-11-13T18:13:21,458 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d095cf21b421409ab1b808fd8798d1cf, entries=150, sequenceid=291, filesize=12.0 K 2024-11-13T18:13:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f5850f3eb01a489080812834455ccea2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f5850f3eb01a489080812834455ccea2 2024-11-13T18:13:21,464 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f5850f3eb01a489080812834455ccea2, entries=150, sequenceid=291, filesize=12.0 K 2024-11-13T18:13:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/da943ba3e98c424699a33a46669e3a61 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da943ba3e98c424699a33a46669e3a61 2024-11-13T18:13:21,476 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da943ba3e98c424699a33a46669e3a61, entries=150, sequenceid=291, filesize=12.0 K 2024-11-13T18:13:21,477 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 5c8c2fb53a8f19068c501d6951c56b68 in 1040ms, sequenceid=291, compaction requested=false 2024-11-13T18:13:21,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:21,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:21,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T18:13:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T18:13:21,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T18:13:21,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6710 sec 2024-11-13T18:13:21,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.6770 sec 2024-11-13T18:13:21,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521600654, max timestamp=1731521601805Num uniques 859; 2024-11-13T18:13:21,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:21,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521600654, max timestamp=1731521601805Num uniques 859; 2024-11-13T18:13:21,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:21,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-13T18:13:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:21,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:21,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:21,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:21,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:21,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:21,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:21,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/6a86f503405b4fa5b7786b0579c3936a is 50, key is test_row_0/A:col10/1731521600664/Put/seqid=0 2024-11-13T18:13:21,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521601841, max timestamp=1731521601855Num uniques 859; 2024-11-13T18:13:21,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:13:21,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521601841, max timestamp=1731521601855Num uniques 859; 2024-11-13T18:13:21,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:13:21,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521601841, max timestamp=1731521601855Num uniques 859; 2024-11-13T18:13:21,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:13:21,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742132_1308 (size=12301) 2024-11-13T18:13:21,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/6a86f503405b4fa5b7786b0579c3936a 2024-11-13T18:13:21,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/8c456227ca6848cc847cf3ed53747815 is 50, key is test_row_0/B:col10/1731521600664/Put/seqid=0 2024-11-13T18:13:21,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521661878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521661880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521661881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521661889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521661892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742133_1309 (size=12301) 2024-11-13T18:13:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:13:21,955 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:21,955 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-13T18:13:21,960 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:13:21,961 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:21,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:21,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521661994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521661994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521661994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521662004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521662004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:13:22,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:13:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:22,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521662204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521662204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521662204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521662215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521662216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:13:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:13:22,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/8c456227ca6848cc847cf3ed53747815 2024-11-13T18:13:22,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/efb0c8905fbf4de98b77e45febfa5e0f is 50, key is test_row_0/C:col10/1731521600664/Put/seqid=0 2024-11-13T18:13:22,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742134_1310 (size=12301) 2024-11-13T18:13:22,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:13:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521662514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521662514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521662516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521662525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521662525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:13:22,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:13:22,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:22,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:13:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:22,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/efb0c8905fbf4de98b77e45febfa5e0f 2024-11-13T18:13:22,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/6a86f503405b4fa5b7786b0579c3936a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6a86f503405b4fa5b7786b0579c3936a 2024-11-13T18:13:22,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6a86f503405b4fa5b7786b0579c3936a, entries=150, sequenceid=315, filesize=12.0 K 2024-11-13T18:13:22,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/8c456227ca6848cc847cf3ed53747815 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8c456227ca6848cc847cf3ed53747815 2024-11-13T18:13:22,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8c456227ca6848cc847cf3ed53747815, entries=150, sequenceid=315, filesize=12.0 K 2024-11-13T18:13:22,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/efb0c8905fbf4de98b77e45febfa5e0f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/efb0c8905fbf4de98b77e45febfa5e0f 2024-11-13T18:13:22,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/efb0c8905fbf4de98b77e45febfa5e0f, entries=150, sequenceid=315, filesize=12.0 K 2024-11-13T18:13:22,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 5c8c2fb53a8f19068c501d6951c56b68 in 1017ms, sequenceid=315, compaction requested=true 2024-11-13T18:13:22,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:22,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:22,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:22,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:22,835 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:22,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:22,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:22,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:22,835 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:22,836 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:22,836 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:22,836 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:22,836 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:22,836 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,836 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,836 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6fd7bc6a81304c7f8327f5e6a14ca782, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f5850f3eb01a489080812834455ccea2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8c456227ca6848cc847cf3ed53747815] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=36.5 K 2024-11-13T18:13:22,837 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/89e26d57f51b42e298a828974112437e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d095cf21b421409ab1b808fd8798d1cf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6a86f503405b4fa5b7786b0579c3936a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=36.5 K 2024-11-13T18:13:22,837 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e26d57f51b42e298a828974112437e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731521599120 2024-11-13T18:13:22,837 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fd7bc6a81304c7f8327f5e6a14ca782, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731521599120 2024-11-13T18:13:22,838 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d095cf21b421409ab1b808fd8798d1cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731521599450 2024-11-13T18:13:22,838 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f5850f3eb01a489080812834455ccea2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731521599450 2024-11-13T18:13:22,838 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c456227ca6848cc847cf3ed53747815, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731521600662 2024-11-13T18:13:22,838 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a86f503405b4fa5b7786b0579c3936a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731521600662 2024-11-13T18:13:22,856 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:22,857 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/38a455c3434f46a9a39fba799f971fd0 is 50, key is test_row_0/A:col10/1731521600664/Put/seqid=0 2024-11-13T18:13:22,873 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:22,874 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/d0b21e6f90734a1eba9dd901f4efc140 is 50, key is test_row_0/B:col10/1731521600664/Put/seqid=0 2024-11-13T18:13:22,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742136_1312 (size=12881) 2024-11-13T18:13:22,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:13:22,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,907 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=127.47 KB heapSize=351 KB 2024-11-13T18:13:22,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:22,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:22,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:22,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:22,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:22,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:22,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742135_1311 (size=12881) 2024-11-13T18:13:22,919 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/d0b21e6f90734a1eba9dd901f4efc140 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/d0b21e6f90734a1eba9dd901f4efc140 2024-11-13T18:13:22,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/7a6a147c96fe46f1a85869cc00f9faa0 is 50, key is test_row_0/A:col10/1731521601876/Put/seqid=0 2024-11-13T18:13:22,926 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/38a455c3434f46a9a39fba799f971fd0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/38a455c3434f46a9a39fba799f971fd0 2024-11-13T18:13:22,927 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into d0b21e6f90734a1eba9dd901f4efc140(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:22,927 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:22,927 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=13, startTime=1731521602835; duration=0sec 2024-11-13T18:13:22,927 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:22,928 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:22,928 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:22,931 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:22,931 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:22,931 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:22,932 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/0133a20649f54e1587938ac639392758, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da943ba3e98c424699a33a46669e3a61, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/efb0c8905fbf4de98b77e45febfa5e0f] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=36.5 K 2024-11-13T18:13:22,933 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 0133a20649f54e1587938ac639392758, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731521599120 2024-11-13T18:13:22,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742137_1313 (size=12301) 2024-11-13T18:13:22,934 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into 38a455c3434f46a9a39fba799f971fd0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:22,934 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:22,934 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=13, startTime=1731521602835; duration=0sec 2024-11-13T18:13:22,934 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:22,934 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:22,934 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting da943ba3e98c424699a33a46669e3a61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731521599450 2024-11-13T18:13:22,934 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/7a6a147c96fe46f1a85869cc00f9faa0 2024-11-13T18:13:22,934 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting efb0c8905fbf4de98b77e45febfa5e0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731521600662 2024-11-13T18:13:22,944 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:22,945 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/ec656bab74d54136a87ebf59f9df0f78 is 50, key is test_row_0/C:col10/1731521600664/Put/seqid=0 2024-11-13T18:13:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/73cda3a35ff546a19d54a0264a1fa775 is 50, key is test_row_0/B:col10/1731521601876/Put/seqid=0 2024-11-13T18:13:22,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742138_1314 (size=12881) 2024-11-13T18:13:22,960 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/ec656bab74d54136a87ebf59f9df0f78 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/ec656bab74d54136a87ebf59f9df0f78 2024-11-13T18:13:22,967 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into ec656bab74d54136a87ebf59f9df0f78(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:22,967 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:22,967 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=13, startTime=1731521602835; duration=0sec 2024-11-13T18:13:22,967 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:22,967 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:22,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742139_1315 (size=12301) 2024-11-13T18:13:22,990 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/73cda3a35ff546a19d54a0264a1fa775 2024-11-13T18:13:23,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a5e9cc1a47c742b4a92d057b61495869 is 50, key is test_row_0/C:col10/1731521601876/Put/seqid=0 2024-11-13T18:13:23,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742140_1316 (size=12301) 2024-11-13T18:13:23,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:23,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521663040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521663042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521663043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521663045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521663045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:13:23,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521663154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521663155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521663155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521663155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521663155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521663364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521663365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521663365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521663374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521663374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,428 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a5e9cc1a47c742b4a92d057b61495869 2024-11-13T18:13:23,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/7a6a147c96fe46f1a85869cc00f9faa0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/7a6a147c96fe46f1a85869cc00f9faa0 2024-11-13T18:13:23,441 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/7a6a147c96fe46f1a85869cc00f9faa0, entries=150, sequenceid=348, filesize=12.0 K 2024-11-13T18:13:23,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/73cda3a35ff546a19d54a0264a1fa775 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/73cda3a35ff546a19d54a0264a1fa775 2024-11-13T18:13:23,446 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/73cda3a35ff546a19d54a0264a1fa775, entries=150, sequenceid=348, filesize=12.0 K 2024-11-13T18:13:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a5e9cc1a47c742b4a92d057b61495869 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5e9cc1a47c742b4a92d057b61495869 2024-11-13T18:13:23,453 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5e9cc1a47c742b4a92d057b61495869, entries=150, sequenceid=348, filesize=12.0 K 2024-11-13T18:13:23,454 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~350.95 KB/359376, currentSize=60.38 KB/61830 for 5c8c2fb53a8f19068c501d6951c56b68 in 547ms, sequenceid=348, compaction requested=false 2024-11-13T18:13:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-13T18:13:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-13T18:13:23,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T18:13:23,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4960 sec 2024-11-13T18:13:23,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.5050 sec 2024-11-13T18:13:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:23,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:13:23,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:23,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:23,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:23,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:23,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:23,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:23,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/088d8f61324c44b187724034ad79db8d is 50, key is test_row_0/A:col10/1731521603039/Put/seqid=0 2024-11-13T18:13:23,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742141_1317 (size=14741) 2024-11-13T18:13:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521663699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521663701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521663701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521663702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521663706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521663814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521663814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521663814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521663814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521663814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521664024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521664024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521664025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521664025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521664025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/088d8f61324c44b187724034ad79db8d 2024-11-13T18:13:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:13:24,104 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:24,105 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-13T18:13:24,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6f3fa3fd45174ad7bc711751e3d7c9ff is 50, key is test_row_0/B:col10/1731521603039/Put/seqid=0 2024-11-13T18:13:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:13:24,110 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:24,111 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:24,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:24,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742142_1318 (size=12301) 2024-11-13T18:13:24,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6f3fa3fd45174ad7bc711751e3d7c9ff 2024-11-13T18:13:24,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/858c2fe0854f46e3919e0fc6cfe77431 is 50, key is test_row_0/C:col10/1731521603039/Put/seqid=0 2024-11-13T18:13:24,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742143_1319 (size=12301) 2024-11-13T18:13:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:13:24,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:13:24,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:24,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:24,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:24,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:24,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521664334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521664334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521664334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521664334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521664344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:13:24,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:24,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:24,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:24,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:13:24,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/858c2fe0854f46e3919e0fc6cfe77431 2024-11-13T18:13:24,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/088d8f61324c44b187724034ad79db8d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/088d8f61324c44b187724034ad79db8d 2024-11-13T18:13:24,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/088d8f61324c44b187724034ad79db8d, entries=200, sequenceid=365, filesize=14.4 K 2024-11-13T18:13:24,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6f3fa3fd45174ad7bc711751e3d7c9ff as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6f3fa3fd45174ad7bc711751e3d7c9ff 2024-11-13T18:13:24,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:13:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:24,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:24,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6f3fa3fd45174ad7bc711751e3d7c9ff, entries=150, sequenceid=365, filesize=12.0 K 2024-11-13T18:13:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/858c2fe0854f46e3919e0fc6cfe77431 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/858c2fe0854f46e3919e0fc6cfe77431 2024-11-13T18:13:24,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/858c2fe0854f46e3919e0fc6cfe77431, entries=150, sequenceid=365, filesize=12.0 K 2024-11-13T18:13:24,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5c8c2fb53a8f19068c501d6951c56b68 in 907ms, sequenceid=365, compaction requested=true 2024-11-13T18:13:24,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:24,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:24,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:24,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:24,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:24,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:24,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:24,583 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:24,583 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:24,584 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:24,584 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:24,584 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:24,584 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:24,584 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:24,584 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:24,584 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/38a455c3434f46a9a39fba799f971fd0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/7a6a147c96fe46f1a85869cc00f9faa0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/088d8f61324c44b187724034ad79db8d] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=39.0 K 2024-11-13T18:13:24,584 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/d0b21e6f90734a1eba9dd901f4efc140, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/73cda3a35ff546a19d54a0264a1fa775, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6f3fa3fd45174ad7bc711751e3d7c9ff] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=36.6 K 2024-11-13T18:13:24,584 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38a455c3434f46a9a39fba799f971fd0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731521600662 2024-11-13T18:13:24,584 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d0b21e6f90734a1eba9dd901f4efc140, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731521600662 2024-11-13T18:13:24,585 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a6a147c96fe46f1a85869cc00f9faa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731521601876 2024-11-13T18:13:24,585 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 73cda3a35ff546a19d54a0264a1fa775, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731521601876 2024-11-13T18:13:24,585 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 088d8f61324c44b187724034ad79db8d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731521603039 2024-11-13T18:13:24,585 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f3fa3fd45174ad7bc711751e3d7c9ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731521603039 2024-11-13T18:13:24,593 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:24,593 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:24,593 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/9631a70204894203bcb4d752417fb3db is 50, key is test_row_0/A:col10/1731521603039/Put/seqid=0 2024-11-13T18:13:24,594 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/709686fc3d2f4acb9ee9d46154bdfd21 is 50, key is test_row_0/B:col10/1731521603039/Put/seqid=0 2024-11-13T18:13:24,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742144_1320 (size=12983) 2024-11-13T18:13:24,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742145_1321 (size=12983) 2024-11-13T18:13:24,629 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/709686fc3d2f4acb9ee9d46154bdfd21 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/709686fc3d2f4acb9ee9d46154bdfd21 2024-11-13T18:13:24,635 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into 709686fc3d2f4acb9ee9d46154bdfd21(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:24,635 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:24,635 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=13, startTime=1731521604583; duration=0sec 2024-11-13T18:13:24,635 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:24,635 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:24,636 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:24,637 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:24,637 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:24,638 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:24,638 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/ec656bab74d54136a87ebf59f9df0f78, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5e9cc1a47c742b4a92d057b61495869, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/858c2fe0854f46e3919e0fc6cfe77431] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=36.6 K 2024-11-13T18:13:24,638 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ec656bab74d54136a87ebf59f9df0f78, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731521600662 2024-11-13T18:13:24,638 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e9cc1a47c742b4a92d057b61495869, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731521601876 2024-11-13T18:13:24,639 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 858c2fe0854f46e3919e0fc6cfe77431, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731521603039 2024-11-13T18:13:24,648 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:24,648 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/34fd725383a44e44a1f2853d6b9967a8 is 50, key is test_row_0/C:col10/1731521603039/Put/seqid=0 2024-11-13T18:13:24,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742146_1322 (size=12983) 2024-11-13T18:13:24,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:13:24,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:24,730 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:13:24,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:24,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:24,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:24,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:24,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:24,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:24,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/4b5e3bd3d38b480f9f0dbe8125295e30 is 50, key is test_row_0/A:col10/1731521603699/Put/seqid=0 2024-11-13T18:13:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:13:24,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742147_1323 (size=12301) 2024-11-13T18:13:24,771 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/4b5e3bd3d38b480f9f0dbe8125295e30 2024-11-13T18:13:24,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6e9753f2825d44f4ae9baadeb9993b62 is 50, key is test_row_0/B:col10/1731521603699/Put/seqid=0 2024-11-13T18:13:24,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742148_1324 (size=12301) 2024-11-13T18:13:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:24,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521664862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521664864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521664870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521664874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521664874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521664984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521664985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521664985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521664985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:24,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:24,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,020 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/9631a70204894203bcb4d752417fb3db as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9631a70204894203bcb4d752417fb3db 2024-11-13T18:13:25,024 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into 9631a70204894203bcb4d752417fb3db(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:25,024 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:25,024 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=13, startTime=1731521604583; duration=0sec 2024-11-13T18:13:25,025 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:25,025 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:25,083 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/34fd725383a44e44a1f2853d6b9967a8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/34fd725383a44e44a1f2853d6b9967a8 2024-11-13T18:13:25,090 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into 34fd725383a44e44a1f2853d6b9967a8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:25,090 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:25,090 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=13, startTime=1731521604583; duration=0sec 2024-11-13T18:13:25,090 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:25,090 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:25,188 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6e9753f2825d44f4ae9baadeb9993b62 2024-11-13T18:13:25,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521665194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/e91c1c42fed0421cbb3f90ff6bb3e038 is 50, key is test_row_0/C:col10/1731521603699/Put/seqid=0 2024-11-13T18:13:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521665194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521665194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742149_1325 (size=12301) 2024-11-13T18:13:25,203 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/e91c1c42fed0421cbb3f90ff6bb3e038 2024-11-13T18:13:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521665204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/4b5e3bd3d38b480f9f0dbe8125295e30 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4b5e3bd3d38b480f9f0dbe8125295e30 2024-11-13T18:13:25,214 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4b5e3bd3d38b480f9f0dbe8125295e30, entries=150, sequenceid=384, filesize=12.0 K 2024-11-13T18:13:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6e9753f2825d44f4ae9baadeb9993b62 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6e9753f2825d44f4ae9baadeb9993b62 2024-11-13T18:13:25,220 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6e9753f2825d44f4ae9baadeb9993b62, entries=150, sequenceid=384, filesize=12.0 K 2024-11-13T18:13:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/e91c1c42fed0421cbb3f90ff6bb3e038 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/e91c1c42fed0421cbb3f90ff6bb3e038 2024-11-13T18:13:25,224 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/e91c1c42fed0421cbb3f90ff6bb3e038, entries=150, sequenceid=384, filesize=12.0 K 2024-11-13T18:13:25,225 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 5c8c2fb53a8f19068c501d6951c56b68 in 495ms, sequenceid=384, compaction requested=false 2024-11-13T18:13:25,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:25,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:25,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-13T18:13:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-13T18:13:25,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T18:13:25,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1190 sec 2024-11-13T18:13:25,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.1330 sec 2024-11-13T18:13:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:13:25,264 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:25,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-13T18:13:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:13:25,268 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:25,269 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:25,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:13:25,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T18:13:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:25,423 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:13:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:25,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:25,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:25,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:25,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/9eaf7e77c0684cc0bb9ca5585a933fef is 50, key is test_row_0/A:col10/1731521604870/Put/seqid=0 2024-11-13T18:13:25,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742150_1326 (size=12301) 2024-11-13T18:13:25,436 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/9eaf7e77c0684cc0bb9ca5585a933fef 2024-11-13T18:13:25,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/edd81cb0428f4ea4ace07dbdd765874b is 50, key is test_row_0/B:col10/1731521604870/Put/seqid=0 2024-11-13T18:13:25,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742151_1327 (size=12301) 2024-11-13T18:13:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:25,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:25,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521665530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521665535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521665536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521665538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:13:25,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521665644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521665644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521665644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521665654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521665854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521665854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521665854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,858 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/edd81cb0428f4ea4ace07dbdd765874b 2024-11-13T18:13:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521665864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d47eaa8d5d2740f58a76eb712cb73eec is 50, key is test_row_0/C:col10/1731521604870/Put/seqid=0 2024-11-13T18:13:25,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742152_1328 (size=12301) 2024-11-13T18:13:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521665884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:25,889 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 5c8c2fb53a8f19068c501d6951c56b68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:13:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:13:26,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521666165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521666165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521666165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521666174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,281 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d47eaa8d5d2740f58a76eb712cb73eec 2024-11-13T18:13:26,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/9eaf7e77c0684cc0bb9ca5585a933fef as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9eaf7e77c0684cc0bb9ca5585a933fef 2024-11-13T18:13:26,293 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9eaf7e77c0684cc0bb9ca5585a933fef, entries=150, sequenceid=401, filesize=12.0 K 2024-11-13T18:13:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/edd81cb0428f4ea4ace07dbdd765874b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/edd81cb0428f4ea4ace07dbdd765874b 2024-11-13T18:13:26,314 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/edd81cb0428f4ea4ace07dbdd765874b, entries=150, sequenceid=401, filesize=12.0 K 2024-11-13T18:13:26,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d47eaa8d5d2740f58a76eb712cb73eec as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d47eaa8d5d2740f58a76eb712cb73eec 2024-11-13T18:13:26,333 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d47eaa8d5d2740f58a76eb712cb73eec, entries=150, sequenceid=401, filesize=12.0 K 2024-11-13T18:13:26,334 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5c8c2fb53a8f19068c501d6951c56b68 in 911ms, sequenceid=401, compaction requested=true 2024-11-13T18:13:26,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:26,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:26,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-13T18:13:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-13T18:13:26,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-13T18:13:26,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0670 sec 2024-11-13T18:13:26,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.0750 sec 2024-11-13T18:13:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:13:26,407 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:26,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-13T18:13:26,411 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:13:26,411 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:26,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:13:26,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T18:13:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:26,565 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:13:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:26,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/cca2679a27154f27aebc9857f176cd2b is 50, key is test_row_0/A:col10/1731521605523/Put/seqid=0 2024-11-13T18:13:26,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742153_1329 (size=12301) 2024-11-13T18:13:26,626 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/cca2679a27154f27aebc9857f176cd2b 2024-11-13T18:13:26,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/aab067557dcc4698a480f9d0bece1570 is 50, key is test_row_0/B:col10/1731521605523/Put/seqid=0 2024-11-13T18:13:26,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742154_1330 (size=12301) 2024-11-13T18:13:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:26,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:26,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521666697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521666702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521666705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521666706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:13:26,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521666815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521666815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521666815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521666815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521667025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521667025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521667025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521667026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:13:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,072 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/aab067557dcc4698a480f9d0bece1570 2024-11-13T18:13:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/57f314cb7ae64fb5bc6cde1fa0ae09e5 is 50, key is test_row_0/C:col10/1731521605523/Put/seqid=0 2024-11-13T18:13:27,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742155_1331 (size=12301) 2024-11-13T18:13:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521667334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521667344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521667344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521667344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,501 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/57f314cb7ae64fb5bc6cde1fa0ae09e5 2024-11-13T18:13:27,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/cca2679a27154f27aebc9857f176cd2b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/cca2679a27154f27aebc9857f176cd2b 2024-11-13T18:13:27,510 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/cca2679a27154f27aebc9857f176cd2b, entries=150, sequenceid=418, filesize=12.0 K 2024-11-13T18:13:27,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/aab067557dcc4698a480f9d0bece1570 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/aab067557dcc4698a480f9d0bece1570 2024-11-13T18:13:27,514 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/aab067557dcc4698a480f9d0bece1570, entries=150, sequenceid=418, filesize=12.0 K 2024-11-13T18:13:27,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/57f314cb7ae64fb5bc6cde1fa0ae09e5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/57f314cb7ae64fb5bc6cde1fa0ae09e5 2024-11-13T18:13:27,519 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/57f314cb7ae64fb5bc6cde1fa0ae09e5, entries=150, sequenceid=418, filesize=12.0 K 2024-11-13T18:13:27,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5c8c2fb53a8f19068c501d6951c56b68 in 955ms, sequenceid=418, compaction requested=true 2024-11-13T18:13:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-13T18:13:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-13T18:13:27,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-13T18:13:27,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1100 sec 2024-11-13T18:13:27,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1160 sec 2024-11-13T18:13:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:13:27,544 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:27,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-13T18:13:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:13:27,546 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:27,547 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:27,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:13:27,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T18:13:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:27,699 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:13:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/54b24cc5fd6d417ea3cbe89987a2f23c is 50, key is test_row_0/A:col10/1731521606703/Put/seqid=0 2024-11-13T18:13:27,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742156_1332 (size=12301) 2024-11-13T18:13:27,730 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/54b24cc5fd6d417ea3cbe89987a2f23c 2024-11-13T18:13:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/de2ef76bf20a474880cf12cd3cf6d499 is 50, key is test_row_0/B:col10/1731521606703/Put/seqid=0 2024-11-13T18:13:27,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742157_1333 (size=12301) 2024-11-13T18:13:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:27,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:13:27,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521667904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,913 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 5c8c2fb53a8f19068c501d6951c56b68 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:13:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521667908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521667908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521667909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521667909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:27,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:27,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521668024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521668024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521668024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521668024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,164 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/de2ef76bf20a474880cf12cd3cf6d499 2024-11-13T18:13:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:13:28,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a9ea4a7cc12b4188ba6e19a1ccf1c0d6 is 50, key is test_row_0/C:col10/1731521606703/Put/seqid=0 2024-11-13T18:13:28,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742158_1334 (size=12301) 2024-11-13T18:13:28,188 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a9ea4a7cc12b4188ba6e19a1ccf1c0d6 2024-11-13T18:13:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/54b24cc5fd6d417ea3cbe89987a2f23c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/54b24cc5fd6d417ea3cbe89987a2f23c 2024-11-13T18:13:28,204 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/54b24cc5fd6d417ea3cbe89987a2f23c, entries=150, sequenceid=433, filesize=12.0 K 2024-11-13T18:13:28,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/de2ef76bf20a474880cf12cd3cf6d499 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/de2ef76bf20a474880cf12cd3cf6d499 2024-11-13T18:13:28,209 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/de2ef76bf20a474880cf12cd3cf6d499, entries=150, sequenceid=433, filesize=12.0 K 2024-11-13T18:13:28,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a9ea4a7cc12b4188ba6e19a1ccf1c0d6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a9ea4a7cc12b4188ba6e19a1ccf1c0d6 2024-11-13T18:13:28,216 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a9ea4a7cc12b4188ba6e19a1ccf1c0d6, entries=150, sequenceid=433, filesize=12.0 K 2024-11-13T18:13:28,217 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5c8c2fb53a8f19068c501d6951c56b68 in 518ms, sequenceid=433, compaction requested=true 2024-11-13T18:13:28,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:28,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:28,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-13T18:13:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-13T18:13:28,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-13T18:13:28,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 672 msec 2024-11-13T18:13:28,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 678 msec 2024-11-13T18:13:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:28,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:13:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:28,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/91d5663e2e7f4a74a68d475130d0fd2c is 50, key is test_row_0/A:col10/1731521607893/Put/seqid=0 2024-11-13T18:13:28,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742159_1335 (size=12301) 2024-11-13T18:13:28,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521668255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521668256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521668256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521668257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521668374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521668374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521668374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521668374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521668584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521668584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521668584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521668585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/91d5663e2e7f4a74a68d475130d0fd2c 2024-11-13T18:13:28,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f2290edeb5ef4c9fb2af1a6c7b17e7de is 50, key is test_row_0/B:col10/1731521607893/Put/seqid=0 2024-11-13T18:13:28,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742160_1336 (size=12301) 2024-11-13T18:13:28,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f2290edeb5ef4c9fb2af1a6c7b17e7de 2024-11-13T18:13:28,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/41465f3125a4454cb3fc1e09c4be663a is 50, key is test_row_0/C:col10/1731521607893/Put/seqid=0 2024-11-13T18:13:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:13:28,684 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:28,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-13T18:13:28,688 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:13:28,689 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:28,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742161_1337 (size=12301) 2024-11-13T18:13:28,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/41465f3125a4454cb3fc1e09c4be663a 2024-11-13T18:13:28,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/91d5663e2e7f4a74a68d475130d0fd2c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/91d5663e2e7f4a74a68d475130d0fd2c 2024-11-13T18:13:28,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/91d5663e2e7f4a74a68d475130d0fd2c, entries=150, sequenceid=452, filesize=12.0 K 2024-11-13T18:13:28,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f2290edeb5ef4c9fb2af1a6c7b17e7de as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f2290edeb5ef4c9fb2af1a6c7b17e7de 2024-11-13T18:13:28,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f2290edeb5ef4c9fb2af1a6c7b17e7de, entries=150, sequenceid=452, filesize=12.0 K 2024-11-13T18:13:28,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/41465f3125a4454cb3fc1e09c4be663a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/41465f3125a4454cb3fc1e09c4be663a 2024-11-13T18:13:28,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/41465f3125a4454cb3fc1e09c4be663a, entries=150, sequenceid=452, filesize=12.0 K 2024-11-13T18:13:28,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5c8c2fb53a8f19068c501d6951c56b68 in 476ms, sequenceid=452, compaction requested=true 2024-11-13T18:13:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:28,714 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-13T18:13:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:28,714 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-13T18:13:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:28,716 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74488 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-13T18:13:28,716 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:28,716 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:28,716 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9631a70204894203bcb4d752417fb3db, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4b5e3bd3d38b480f9f0dbe8125295e30, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9eaf7e77c0684cc0bb9ca5585a933fef, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/cca2679a27154f27aebc9857f176cd2b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/54b24cc5fd6d417ea3cbe89987a2f23c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/91d5663e2e7f4a74a68d475130d0fd2c] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=72.7 K 2024-11-13T18:13:28,716 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74488 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-13T18:13:28,716 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9631a70204894203bcb4d752417fb3db, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731521603039 2024-11-13T18:13:28,717 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:28,717 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:28,717 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/709686fc3d2f4acb9ee9d46154bdfd21, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6e9753f2825d44f4ae9baadeb9993b62, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/edd81cb0428f4ea4ace07dbdd765874b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/aab067557dcc4698a480f9d0bece1570, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/de2ef76bf20a474880cf12cd3cf6d499, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f2290edeb5ef4c9fb2af1a6c7b17e7de] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=72.7 K 2024-11-13T18:13:28,717 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b5e3bd3d38b480f9f0dbe8125295e30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731521603699 2024-11-13T18:13:28,717 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 709686fc3d2f4acb9ee9d46154bdfd21, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731521603039 2024-11-13T18:13:28,717 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eaf7e77c0684cc0bb9ca5585a933fef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731521604851 2024-11-13T18:13:28,717 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e9753f2825d44f4ae9baadeb9993b62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731521603699 2024-11-13T18:13:28,718 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting cca2679a27154f27aebc9857f176cd2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731521605523 2024-11-13T18:13:28,718 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting edd81cb0428f4ea4ace07dbdd765874b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731521604851 2024-11-13T18:13:28,718 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54b24cc5fd6d417ea3cbe89987a2f23c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731521606699 2024-11-13T18:13:28,718 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting aab067557dcc4698a480f9d0bece1570, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731521605523 2024-11-13T18:13:28,718 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91d5663e2e7f4a74a68d475130d0fd2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731521607893 2024-11-13T18:13:28,718 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting de2ef76bf20a474880cf12cd3cf6d499, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731521606699 2024-11-13T18:13:28,719 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f2290edeb5ef4c9fb2af1a6c7b17e7de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731521607893 2024-11-13T18:13:28,737 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#288 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:28,739 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/e6753fc119a8462ba7be3a97a91a3938 is 50, key is test_row_0/A:col10/1731521607893/Put/seqid=0 2024-11-13T18:13:28,740 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#289 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:28,741 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/7f9d742e4a1d4d2096bf3d1a0a90b62d is 50, key is test_row_0/B:col10/1731521607893/Put/seqid=0 2024-11-13T18:13:28,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742162_1338 (size=13187) 2024-11-13T18:13:28,756 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/e6753fc119a8462ba7be3a97a91a3938 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e6753fc119a8462ba7be3a97a91a3938 2024-11-13T18:13:28,763 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into e6753fc119a8462ba7be3a97a91a3938(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:28,763 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:28,763 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=10, startTime=1731521608714; duration=0sec 2024-11-13T18:13:28,763 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:28,763 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:28,763 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-13T18:13:28,767 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74488 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-13T18:13:28,767 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:28,767 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:28,767 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/34fd725383a44e44a1f2853d6b9967a8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/e91c1c42fed0421cbb3f90ff6bb3e038, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d47eaa8d5d2740f58a76eb712cb73eec, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/57f314cb7ae64fb5bc6cde1fa0ae09e5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a9ea4a7cc12b4188ba6e19a1ccf1c0d6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/41465f3125a4454cb3fc1e09c4be663a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=72.7 K 2024-11-13T18:13:28,768 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34fd725383a44e44a1f2853d6b9967a8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731521603039 2024-11-13T18:13:28,768 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e91c1c42fed0421cbb3f90ff6bb3e038, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731521603699 2024-11-13T18:13:28,769 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47eaa8d5d2740f58a76eb712cb73eec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731521604851 2024-11-13T18:13:28,769 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57f314cb7ae64fb5bc6cde1fa0ae09e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731521605523 2024-11-13T18:13:28,769 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9ea4a7cc12b4188ba6e19a1ccf1c0d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731521606699 2024-11-13T18:13:28,770 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41465f3125a4454cb3fc1e09c4be663a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731521607893 2024-11-13T18:13:28,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742163_1339 (size=13187) 2024-11-13T18:13:28,788 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#290 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:28,789 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/0b5765bf69d3400d885ba95815c3adac is 50, key is test_row_0/C:col10/1731521607893/Put/seqid=0 2024-11-13T18:13:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:13:28,794 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/7f9d742e4a1d4d2096bf3d1a0a90b62d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7f9d742e4a1d4d2096bf3d1a0a90b62d 2024-11-13T18:13:28,805 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into 7f9d742e4a1d4d2096bf3d1a0a90b62d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:28,805 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:28,805 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=10, startTime=1731521608714; duration=0sec 2024-11-13T18:13:28,806 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:28,806 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:28,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742164_1340 (size=13187) 2024-11-13T18:13:28,830 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/0b5765bf69d3400d885ba95815c3adac as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/0b5765bf69d3400d885ba95815c3adac 2024-11-13T18:13:28,842 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into 0b5765bf69d3400d885ba95815c3adac(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:28,842 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:28,842 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=10, startTime=1731521608714; duration=0sec 2024-11-13T18:13:28,842 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:28,842 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:28,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:13:28,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:28,846 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:13:28,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:28,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:28,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:28,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:28,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:28,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:28,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/1bf407e00acf48398c6593778e866307 is 50, key is test_row_0/A:col10/1731521608254/Put/seqid=0 2024-11-13T18:13:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:28,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742165_1341 (size=12301) 2024-11-13T18:13:28,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521668934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731521608920, max timestamp=1731521608939Num uniques 859; 2024-11-13T18:13:28,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:13:28,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521608920, max timestamp=1731521608939Num uniques 859; 2024-11-13T18:13:28,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521608920, max timestamp=1731521608939Num uniques 859; 2024-11-13T18:13:28,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:13:28,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:13:28,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521668973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521668983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521668984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:13:29,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521669044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521669094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521669095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521669095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521669254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,303 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/1bf407e00acf48398c6593778e866307 2024-11-13T18:13:29,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521669304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521669305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521669305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:13:29,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f81617c518654eb4a095c01a8a5d980d is 50, key is test_row_0/B:col10/1731521608254/Put/seqid=0 2024-11-13T18:13:29,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742166_1342 (size=12301) 2024-11-13T18:13:29,377 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f81617c518654eb4a095c01a8a5d980d 2024-11-13T18:13:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/8462825c7cd74824bf553bcd16222f8a is 50, key is test_row_0/C:col10/1731521608254/Put/seqid=0 2024-11-13T18:13:29,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742167_1343 (size=12301) 2024-11-13T18:13:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521669565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521669617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521669617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521669617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:13:29,874 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/8462825c7cd74824bf553bcd16222f8a 2024-11-13T18:13:29,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/1bf407e00acf48398c6593778e866307 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1bf407e00acf48398c6593778e866307 2024-11-13T18:13:29,893 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1bf407e00acf48398c6593778e866307, entries=150, sequenceid=468, filesize=12.0 K 2024-11-13T18:13:29,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f81617c518654eb4a095c01a8a5d980d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f81617c518654eb4a095c01a8a5d980d 2024-11-13T18:13:29,899 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f81617c518654eb4a095c01a8a5d980d, entries=150, sequenceid=468, filesize=12.0 K 2024-11-13T18:13:29,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/8462825c7cd74824bf553bcd16222f8a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8462825c7cd74824bf553bcd16222f8a 2024-11-13T18:13:29,905 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8462825c7cd74824bf553bcd16222f8a, entries=150, sequenceid=468, filesize=12.0 K 2024-11-13T18:13:29,907 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=120.76 KB/123660 for 5c8c2fb53a8f19068c501d6951c56b68 in 1061ms, sequenceid=468, compaction requested=false 2024-11-13T18:13:29,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:29,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:29,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-13T18:13:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-13T18:13:29,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-13T18:13:29,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2200 sec 2024-11-13T18:13:29,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.2260 sec 2024-11-13T18:13:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:30,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T18:13:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:30,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/4861b5f884014434b67b5e0b5e8d83d3 is 50, key is test_row_0/A:col10/1731521608966/Put/seqid=0 2024-11-13T18:13:30,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742168_1344 (size=12301) 2024-11-13T18:13:30,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521670134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:30,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521670134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:30,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521670135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:30,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:30,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521670138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:30,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521670245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:30,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521670254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:30,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521670453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:30,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521670464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:30,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/4861b5f884014434b67b5e0b5e8d83d3 2024-11-13T18:13:30,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/e6766d27fe554522a0bdb6030acc73a4 is 50, key is test_row_0/B:col10/1731521608966/Put/seqid=0 2024-11-13T18:13:30,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742169_1345 (size=12301) 2024-11-13T18:13:30,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/e6766d27fe554522a0bdb6030acc73a4 2024-11-13T18:13:30,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/5332274a2f914757b8e402e5c1850d53 is 50, key is test_row_0/C:col10/1731521608966/Put/seqid=0 2024-11-13T18:13:30,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742170_1346 (size=12301) 2024-11-13T18:13:30,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521670765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:30,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521670773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:13:30,835 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:30,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-13T18:13:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:13:30,841 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:30,842 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:30,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:13:30,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:13:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:30,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:31,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/5332274a2f914757b8e402e5c1850d53 2024-11-13T18:13:31,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/4861b5f884014434b67b5e0b5e8d83d3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4861b5f884014434b67b5e0b5e8d83d3 2024-11-13T18:13:31,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4861b5f884014434b67b5e0b5e8d83d3, entries=150, sequenceid=503, filesize=12.0 K 2024-11-13T18:13:31,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/e6766d27fe554522a0bdb6030acc73a4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e6766d27fe554522a0bdb6030acc73a4 2024-11-13T18:13:31,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e6766d27fe554522a0bdb6030acc73a4, entries=150, sequenceid=503, filesize=12.0 K 2024-11-13T18:13:31,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/5332274a2f914757b8e402e5c1850d53 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/5332274a2f914757b8e402e5c1850d53 2024-11-13T18:13:31,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/5332274a2f914757b8e402e5c1850d53, entries=150, sequenceid=503, filesize=12.0 K 2024-11-13T18:13:31,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 5c8c2fb53a8f19068c501d6951c56b68 in 977ms, sequenceid=503, compaction requested=true 2024-11-13T18:13:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:31,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:31,066 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:31,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:31,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:31,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:31,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:31,066 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:31,066 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:31,066 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:31,067 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:31,067 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e6753fc119a8462ba7be3a97a91a3938, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1bf407e00acf48398c6593778e866307, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4861b5f884014434b67b5e0b5e8d83d3] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=36.9 K 2024-11-13T18:13:31,067 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:31,067 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:31,067 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:31,067 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7f9d742e4a1d4d2096bf3d1a0a90b62d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f81617c518654eb4a095c01a8a5d980d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e6766d27fe554522a0bdb6030acc73a4] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=36.9 K 2024-11-13T18:13:31,067 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6753fc119a8462ba7be3a97a91a3938, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731521607893 2024-11-13T18:13:31,067 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f9d742e4a1d4d2096bf3d1a0a90b62d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731521607893 2024-11-13T18:13:31,067 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bf407e00acf48398c6593778e866307, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731521608246 2024-11-13T18:13:31,068 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f81617c518654eb4a095c01a8a5d980d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731521608246 2024-11-13T18:13:31,068 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4861b5f884014434b67b5e0b5e8d83d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731521608966 2024-11-13T18:13:31,068 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e6766d27fe554522a0bdb6030acc73a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731521608966 2024-11-13T18:13:31,095 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:31,095 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/577585accb4643cbb92cc4460e7b65c2 is 50, key is test_row_0/A:col10/1731521608966/Put/seqid=0 2024-11-13T18:13:31,103 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:31,103 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/8753023cd27a4762815ebc8fa0d0ef67 is 50, key is test_row_0/B:col10/1731521608966/Put/seqid=0 2024-11-13T18:13:31,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742171_1347 (size=13289) 2024-11-13T18:13:31,126 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/577585accb4643cbb92cc4460e7b65c2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/577585accb4643cbb92cc4460e7b65c2 2024-11-13T18:13:31,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742172_1348 (size=13289) 2024-11-13T18:13:31,132 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into 577585accb4643cbb92cc4460e7b65c2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:31,133 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:31,133 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=13, startTime=1731521611065; duration=0sec 2024-11-13T18:13:31,133 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:31,133 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:31,133 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:31,134 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:31,134 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:31,134 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:31,134 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/0b5765bf69d3400d885ba95815c3adac, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8462825c7cd74824bf553bcd16222f8a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/5332274a2f914757b8e402e5c1850d53] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=36.9 K 2024-11-13T18:13:31,135 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b5765bf69d3400d885ba95815c3adac, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731521607893 2024-11-13T18:13:31,135 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8462825c7cd74824bf553bcd16222f8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731521608246 2024-11-13T18:13:31,136 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5332274a2f914757b8e402e5c1850d53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731521608966 2024-11-13T18:13:31,149 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:31,150 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/482afb75a28a4f6cbb1bc91bf1e97d65 is 50, key is test_row_0/C:col10/1731521608966/Put/seqid=0 2024-11-13T18:13:31,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:13:31,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:31,151 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:13:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:31,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:13:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/808393b7a9264105907518dc7440a2b3 is 50, key is test_row_0/A:col10/1731521610137/Put/seqid=0 2024-11-13T18:13:31,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742173_1349 (size=13289) 2024-11-13T18:13:31,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742174_1350 (size=14741) 2024-11-13T18:13:31,183 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/808393b7a9264105907518dc7440a2b3 2024-11-13T18:13:31,191 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/482afb75a28a4f6cbb1bc91bf1e97d65 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/482afb75a28a4f6cbb1bc91bf1e97d65 2024-11-13T18:13:31,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/bb62b171d4db4d1482bd34f6c18854e3 is 50, key is test_row_0/B:col10/1731521610137/Put/seqid=0 2024-11-13T18:13:31,199 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into 482afb75a28a4f6cbb1bc91bf1e97d65(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:31,199 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:31,199 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=13, startTime=1731521611066; duration=0sec 2024-11-13T18:13:31,199 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:31,199 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:31,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742175_1351 (size=12301) 2024-11-13T18:13:31,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/bb62b171d4db4d1482bd34f6c18854e3 2024-11-13T18:13:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521671218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521671221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/4f32c072e848483f8354cda0d1d4ab4b is 50, key is test_row_0/C:col10/1731521610137/Put/seqid=0 2024-11-13T18:13:31,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742176_1352 (size=12301) 2024-11-13T18:13:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521671274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521671284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521671334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521671334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:13:31,541 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/8753023cd27a4762815ebc8fa0d0ef67 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8753023cd27a4762815ebc8fa0d0ef67 2024-11-13T18:13:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521671544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521671544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,553 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into 8753023cd27a4762815ebc8fa0d0ef67(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:31,553 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:31,553 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=13, startTime=1731521611066; duration=0sec 2024-11-13T18:13:31,553 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:31,553 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:31,659 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/4f32c072e848483f8354cda0d1d4ab4b 2024-11-13T18:13:31,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/808393b7a9264105907518dc7440a2b3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/808393b7a9264105907518dc7440a2b3 2024-11-13T18:13:31,713 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/808393b7a9264105907518dc7440a2b3, entries=200, sequenceid=516, filesize=14.4 K 2024-11-13T18:13:31,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/bb62b171d4db4d1482bd34f6c18854e3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/bb62b171d4db4d1482bd34f6c18854e3 2024-11-13T18:13:31,739 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/bb62b171d4db4d1482bd34f6c18854e3, entries=150, sequenceid=516, filesize=12.0 K 2024-11-13T18:13:31,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/4f32c072e848483f8354cda0d1d4ab4b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/4f32c072e848483f8354cda0d1d4ab4b 2024-11-13T18:13:31,762 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/4f32c072e848483f8354cda0d1d4ab4b, entries=150, sequenceid=516, filesize=12.0 K 2024-11-13T18:13:31,763 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5c8c2fb53a8f19068c501d6951c56b68 in 612ms, sequenceid=516, compaction requested=false 2024-11-13T18:13:31,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:31,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:31,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-13T18:13:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-13T18:13:31,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-13T18:13:31,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 929 msec 2024-11-13T18:13:31,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 940 msec 2024-11-13T18:13:31,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521611201, max timestamp=1731521611857Num uniques 859; 2024-11-13T18:13:31,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:31,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-13T18:13:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:31,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:31,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/2713af87524f4a9cb3731eabc8926609 is 50, key is test_row_0/A:col10/1731521611219/Put/seqid=0 2024-11-13T18:13:31,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742177_1353 (size=14741) 2024-11-13T18:13:31,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521671936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521671942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51194 deadline: 1731521671954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:31,957 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 5c8c2fb53a8f19068c501d6951c56b68 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:13:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:31,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:13:31,974 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:31,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-13T18:13:31,976 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:13:31,981 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:31,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:32,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521672054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521672056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:13:32,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T18:13:32,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:32,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521672264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521672264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T18:13:32,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:32,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:13:32,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51160 deadline: 1731521672294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,297 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 5c8c2fb53a8f19068c501d6951c56b68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:13:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1731521672304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,305 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_2' in 5c8c2fb53a8f19068c501d6951c56b68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:13:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/2713af87524f4a9cb3731eabc8926609 2024-11-13T18:13:32,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/4a97523386774a65a1dc7c620d24b8b1 is 50, key is test_row_0/B:col10/1731521611219/Put/seqid=0 2024-11-13T18:13:32,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742178_1354 (size=12301) 2024-11-13T18:13:32,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/4a97523386774a65a1dc7c620d24b8b1 2024-11-13T18:13:32,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/23de1dd4a4c244a3a00eb2791a695e4e is 50, key is test_row_0/C:col10/1731521611219/Put/seqid=0 2024-11-13T18:13:32,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742179_1355 (size=12301) 2024-11-13T18:13:32,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/23de1dd4a4c244a3a00eb2791a695e4e 2024-11-13T18:13:32,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/2713af87524f4a9cb3731eabc8926609 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/2713af87524f4a9cb3731eabc8926609 2024-11-13T18:13:32,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/2713af87524f4a9cb3731eabc8926609, entries=200, sequenceid=539, filesize=14.4 K 2024-11-13T18:13:32,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/4a97523386774a65a1dc7c620d24b8b1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/4a97523386774a65a1dc7c620d24b8b1 2024-11-13T18:13:32,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/4a97523386774a65a1dc7c620d24b8b1, entries=150, sequenceid=539, filesize=12.0 K 2024-11-13T18:13:32,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/23de1dd4a4c244a3a00eb2791a695e4e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/23de1dd4a4c244a3a00eb2791a695e4e 2024-11-13T18:13:32,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/23de1dd4a4c244a3a00eb2791a695e4e, entries=150, sequenceid=539, filesize=12.0 K 2024-11-13T18:13:32,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 5c8c2fb53a8f19068c501d6951c56b68 in 530ms, sequenceid=539, compaction requested=true 2024-11-13T18:13:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:32,393 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:13:32,397 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:32,397 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:32,397 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,398 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/577585accb4643cbb92cc4460e7b65c2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/808393b7a9264105907518dc7440a2b3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/2713af87524f4a9cb3731eabc8926609] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=41.8 K 2024-11-13T18:13:32,398 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 577585accb4643cbb92cc4460e7b65c2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731521608966 2024-11-13T18:13:32,398 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 808393b7a9264105907518dc7440a2b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731521610100 2024-11-13T18:13:32,399 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2713af87524f4a9cb3731eabc8926609, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731521611201 2024-11-13T18:13:32,401 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:32,403 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:32,403 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:32,403 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,403 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8753023cd27a4762815ebc8fa0d0ef67, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/bb62b171d4db4d1482bd34f6c18854e3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/4a97523386774a65a1dc7c620d24b8b1] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=37.0 K 2024-11-13T18:13:32,404 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 8753023cd27a4762815ebc8fa0d0ef67, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731521608966 2024-11-13T18:13:32,405 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting bb62b171d4db4d1482bd34f6c18854e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731521610100 2024-11-13T18:13:32,405 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a97523386774a65a1dc7c620d24b8b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731521611216 2024-11-13T18:13:32,409 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:32,410 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/439f84f1c90f428aa4791b3e5afbdbc9 is 50, key is test_row_0/A:col10/1731521611219/Put/seqid=0 2024-11-13T18:13:32,421 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:32,422 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/9a166fe345b145e782a2aa4b3fb9b18f is 50, key is test_row_0/B:col10/1731521611219/Put/seqid=0 2024-11-13T18:13:32,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T18:13:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,441 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:13:32,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:32,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:32,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:32,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:32,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:32,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:32,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742180_1356 (size=13391) 2024-11-13T18:13:32,468 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/439f84f1c90f428aa4791b3e5afbdbc9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/439f84f1c90f428aa4791b3e5afbdbc9 2024-11-13T18:13:32,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/f9cff9ada3fc46fa91c87b7f2b2e4e14 is 50, key is test_row_0/A:col10/1731521611935/Put/seqid=0 2024-11-13T18:13:32,475 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into 439f84f1c90f428aa4791b3e5afbdbc9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:32,475 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:32,475 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=13, startTime=1731521612393; duration=0sec 2024-11-13T18:13:32,475 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:32,475 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:32,475 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:32,478 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:32,478 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:32,478 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,479 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/482afb75a28a4f6cbb1bc91bf1e97d65, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/4f32c072e848483f8354cda0d1d4ab4b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/23de1dd4a4c244a3a00eb2791a695e4e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=37.0 K 2024-11-13T18:13:32,480 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 482afb75a28a4f6cbb1bc91bf1e97d65, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731521608966 2024-11-13T18:13:32,480 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f32c072e848483f8354cda0d1d4ab4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731521610100 2024-11-13T18:13:32,481 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23de1dd4a4c244a3a00eb2791a695e4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731521611216 2024-11-13T18:13:32,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742181_1357 (size=13391) 2024-11-13T18:13:32,503 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:32,504 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/cecfb098c2bf4b24864379542f1dbc30 is 50, key is test_row_0/C:col10/1731521611219/Put/seqid=0 2024-11-13T18:13:32,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742182_1358 (size=12301) 2024-11-13T18:13:32,508 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/f9cff9ada3fc46fa91c87b7f2b2e4e14 2024-11-13T18:13:32,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742183_1359 (size=13391) 2024-11-13T18:13:32,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/14e03172455a428c8daa1b86ab7da42c is 50, key is test_row_0/B:col10/1731521611935/Put/seqid=0 2024-11-13T18:13:32,534 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/cecfb098c2bf4b24864379542f1dbc30 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/cecfb098c2bf4b24864379542f1dbc30 2024-11-13T18:13:32,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742184_1360 (size=12301) 2024-11-13T18:13:32,539 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/14e03172455a428c8daa1b86ab7da42c 2024-11-13T18:13:32,540 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into cecfb098c2bf4b24864379542f1dbc30(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:32,540 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:32,540 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=13, startTime=1731521612393; duration=0sec 2024-11-13T18:13:32,540 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:32,540 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:32,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/58eb3f5bd3604ccea2e0a1117315ac39 is 50, key is test_row_0/C:col10/1731521611935/Put/seqid=0 2024-11-13T18:13:32,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742185_1361 (size=12301) 2024-11-13T18:13:32,552 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/58eb3f5bd3604ccea2e0a1117315ac39 2024-11-13T18:13:32,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/f9cff9ada3fc46fa91c87b7f2b2e4e14 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f9cff9ada3fc46fa91c87b7f2b2e4e14 2024-11-13T18:13:32,567 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f9cff9ada3fc46fa91c87b7f2b2e4e14, entries=150, sequenceid=555, filesize=12.0 K 2024-11-13T18:13:32,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/14e03172455a428c8daa1b86ab7da42c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/14e03172455a428c8daa1b86ab7da42c 2024-11-13T18:13:32,571 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/14e03172455a428c8daa1b86ab7da42c, entries=150, sequenceid=555, filesize=12.0 K 2024-11-13T18:13:32,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/58eb3f5bd3604ccea2e0a1117315ac39 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/58eb3f5bd3604ccea2e0a1117315ac39 2024-11-13T18:13:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:32,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:32,577 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/58eb3f5bd3604ccea2e0a1117315ac39, entries=150, sequenceid=555, filesize=12.0 K 2024-11-13T18:13:32,578 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for 5c8c2fb53a8f19068c501d6951c56b68 in 137ms, sequenceid=555, compaction requested=false 2024-11-13T18:13:32,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:32,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-13T18:13:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-13T18:13:32,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-13T18:13:32,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 599 msec 2024-11-13T18:13:32,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 608 msec 2024-11-13T18:13:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:32,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:13:32,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:32,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:32,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:32,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:32,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:32,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:32,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/c84f53d0d35744988e530376eff63457 is 50, key is test_row_0/A:col10/1731521612584/Put/seqid=0 2024-11-13T18:13:32,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742186_1362 (size=14741) 2024-11-13T18:13:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:13:32,604 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:32,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-13T18:13:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T18:13:32,606 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:32,607 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:32,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:32,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521612619, max timestamp=1731521612640Num uniques 859; 2024-11-13T18:13:32,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:32,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521612619, max timestamp=1731521612640Num uniques 859; 2024-11-13T18:13:32,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:32,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521672667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521672670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T18:13:32,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T18:13:32,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:32,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521672773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521672784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,904 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/9a166fe345b145e782a2aa4b3fb9b18f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/9a166fe345b145e782a2aa4b3fb9b18f 2024-11-13T18:13:32,909 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into 9a166fe345b145e782a2aa4b3fb9b18f(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:32,909 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:32,909 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=13, startTime=1731521612393; duration=0sec 2024-11-13T18:13:32,909 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:32,909 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:32,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T18:13:32,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:32,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:32,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T18:13:32,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521672984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:32,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521672993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:32,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/c84f53d0d35744988e530376eff63457 2024-11-13T18:13:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:33,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6bca145f42594a06af264d687931b925 is 50, key is test_row_0/B:col10/1731521612584/Put/seqid=0 2024-11-13T18:13:33,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742187_1363 (size=12301) 2024-11-13T18:13:33,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6bca145f42594a06af264d687931b925 2024-11-13T18:13:33,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a48fe1a41bfc4336b773ddb3db8f8365 is 50, key is test_row_0/C:col10/1731521612584/Put/seqid=0 2024-11-13T18:13:33,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742188_1364 (size=12301) 2024-11-13T18:13:33,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a48fe1a41bfc4336b773ddb3db8f8365 2024-11-13T18:13:33,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/c84f53d0d35744988e530376eff63457 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c84f53d0d35744988e530376eff63457 2024-11-13T18:13:33,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c84f53d0d35744988e530376eff63457, entries=200, sequenceid=568, filesize=14.4 K 2024-11-13T18:13:33,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/6bca145f42594a06af264d687931b925 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6bca145f42594a06af264d687931b925 2024-11-13T18:13:33,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6bca145f42594a06af264d687931b925, entries=150, sequenceid=568, filesize=12.0 K 2024-11-13T18:13:33,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/a48fe1a41bfc4336b773ddb3db8f8365 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a48fe1a41bfc4336b773ddb3db8f8365 2024-11-13T18:13:33,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a48fe1a41bfc4336b773ddb3db8f8365, entries=150, sequenceid=568, filesize=12.0 K 2024-11-13T18:13:33,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 5c8c2fb53a8f19068c501d6951c56b68 in 468ms, sequenceid=568, compaction requested=true 2024-11-13T18:13:33,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:33,056 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:33,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:33,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:33,058 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:33,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:33,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:33,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:33,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:33,059 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:33,059 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:33,059 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:33,059 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/439f84f1c90f428aa4791b3e5afbdbc9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f9cff9ada3fc46fa91c87b7f2b2e4e14, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c84f53d0d35744988e530376eff63457] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=39.5 K 2024-11-13T18:13:33,059 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:33,059 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 439f84f1c90f428aa4791b3e5afbdbc9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731521611216 2024-11-13T18:13:33,059 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:33,059 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:33,059 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/9a166fe345b145e782a2aa4b3fb9b18f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/14e03172455a428c8daa1b86ab7da42c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6bca145f42594a06af264d687931b925] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=37.1 K 2024-11-13T18:13:33,060 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9cff9ada3fc46fa91c87b7f2b2e4e14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1731521611869 2024-11-13T18:13:33,060 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c84f53d0d35744988e530376eff63457, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731521612578 2024-11-13T18:13:33,060 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a166fe345b145e782a2aa4b3fb9b18f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731521611216 2024-11-13T18:13:33,060 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 14e03172455a428c8daa1b86ab7da42c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1731521611869 2024-11-13T18:13:33,061 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bca145f42594a06af264d687931b925, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731521612578 2024-11-13T18:13:33,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T18:13:33,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:33,066 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=125.23 KB heapSize=363.39 KB 2024-11-13T18:13:33,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:33,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:33,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:33,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:33,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:33,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:33,069 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:33,070 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/d8426a0c17b546f0a62f2a2d6820fba3 is 50, key is test_row_0/A:col10/1731521612584/Put/seqid=0 2024-11-13T18:13:33,072 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:33,073 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/fdb3ccad3c2b4db284352437c8e03dbe is 50, key is test_row_0/B:col10/1731521612584/Put/seqid=0 2024-11-13T18:13:33,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/0c8d4c18f8ff4b8c97f5a374729eb72e is 50, key is test_row_0/A:col10/1731521612668/Put/seqid=0 2024-11-13T18:13:33,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742189_1365 (size=13493) 2024-11-13T18:13:33,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742190_1366 (size=13493) 2024-11-13T18:13:33,090 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/d8426a0c17b546f0a62f2a2d6820fba3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d8426a0c17b546f0a62f2a2d6820fba3 2024-11-13T18:13:33,097 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/fdb3ccad3c2b4db284352437c8e03dbe as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/fdb3ccad3c2b4db284352437c8e03dbe 2024-11-13T18:13:33,098 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into d8426a0c17b546f0a62f2a2d6820fba3(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:33,098 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:33,098 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=13, startTime=1731521613056; duration=0sec 2024-11-13T18:13:33,098 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:33,098 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:33,098 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:33,099 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:33,099 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:33,099 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:33,099 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/cecfb098c2bf4b24864379542f1dbc30, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/58eb3f5bd3604ccea2e0a1117315ac39, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a48fe1a41bfc4336b773ddb3db8f8365] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=37.1 K 2024-11-13T18:13:33,100 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting cecfb098c2bf4b24864379542f1dbc30, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731521611216 2024-11-13T18:13:33,100 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58eb3f5bd3604ccea2e0a1117315ac39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1731521611869 2024-11-13T18:13:33,101 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48fe1a41bfc4336b773ddb3db8f8365, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731521612578 2024-11-13T18:13:33,103 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into fdb3ccad3c2b4db284352437c8e03dbe(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:33,103 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:33,103 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=13, startTime=1731521613058; duration=0sec 2024-11-13T18:13:33,103 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:33,103 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:33,112 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:33,113 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b048535b9e37402ab6dae50cd4410fd3 is 50, key is test_row_0/C:col10/1731521612584/Put/seqid=0 2024-11-13T18:13:33,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742191_1367 (size=12301) 2024-11-13T18:13:33,114 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/0c8d4c18f8ff4b8c97f5a374729eb72e 2024-11-13T18:13:33,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742192_1368 (size=13493) 2024-11-13T18:13:33,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/67714b02842642aaae5362c2bdedf15f is 50, key is test_row_0/B:col10/1731521612668/Put/seqid=0 2024-11-13T18:13:33,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742193_1369 (size=12301) 2024-11-13T18:13:33,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T18:13:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:33,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. as already flushing 2024-11-13T18:13:33,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521673340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:33,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:33,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:33,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:33,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:33,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521673341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51218 deadline: 1731521673454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:33,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1731521673454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:13:33,488 INFO [Thread-1209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1209. 2024-11-13T18:13:33,488 DEBUG [Thread-1209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:33,488 DEBUG [Thread-1209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,488 DEBUG [Thread-1209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:33,488 INFO [Thread-1207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1207. 2024-11-13T18:13:33,488 DEBUG [Thread-1207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:33,488 DEBUG [Thread-1207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,488 DEBUG [Thread-1207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,491 INFO [Thread-1205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1205. 2024-11-13T18:13:33,491 DEBUG [Thread-1205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:33,491 DEBUG [Thread-1205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,491 DEBUG [Thread-1205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,491 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:33,495 INFO [Thread-1203 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1203. 2024-11-13T18:13:33,495 DEBUG [Thread-1203 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:33,495 DEBUG [Thread-1203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,495 DEBUG [Thread-1203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:33,496 INFO [Thread-1211 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1211. 2024-11-13T18:13:33,496 DEBUG [Thread-1211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:33,496 DEBUG [Thread-1211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,496 DEBUG [Thread-1211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:33,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:33,524 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/b048535b9e37402ab6dae50cd4410fd3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b048535b9e37402ab6dae50cd4410fd3 2024-11-13T18:13:33,530 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into b048535b9e37402ab6dae50cd4410fd3(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:33,530 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:33,530 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=13, startTime=1731521613058; duration=0sec 2024-11-13T18:13:33,530 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:33,530 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:33,539 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/67714b02842642aaae5362c2bdedf15f 2024-11-13T18:13:33,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d2cf9dd7b8b64a4fb6b7a7a9a6e5f8f8 is 50, key is test_row_0/C:col10/1731521612668/Put/seqid=0 2024-11-13T18:13:33,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742194_1370 (size=12301) 2024-11-13T18:13:33,563 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d2cf9dd7b8b64a4fb6b7a7a9a6e5f8f8 2024-11-13T18:13:33,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/0c8d4c18f8ff4b8c97f5a374729eb72e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0c8d4c18f8ff4b8c97f5a374729eb72e 2024-11-13T18:13:33,571 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0c8d4c18f8ff4b8c97f5a374729eb72e, entries=150, sequenceid=600, filesize=12.0 K 2024-11-13T18:13:33,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/67714b02842642aaae5362c2bdedf15f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/67714b02842642aaae5362c2bdedf15f 2024-11-13T18:13:33,575 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/67714b02842642aaae5362c2bdedf15f, entries=150, sequenceid=600, filesize=12.0 K 2024-11-13T18:13:33,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/d2cf9dd7b8b64a4fb6b7a7a9a6e5f8f8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d2cf9dd7b8b64a4fb6b7a7a9a6e5f8f8 2024-11-13T18:13:33,581 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d2cf9dd7b8b64a4fb6b7a7a9a6e5f8f8, entries=150, sequenceid=600, filesize=12.0 K 2024-11-13T18:13:33,593 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~363.34 KB/372064, currentSize=53.67 KB/54960 for 5c8c2fb53a8f19068c501d6951c56b68 in 527ms, sequenceid=600, compaction requested=false 2024-11-13T18:13:33,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:33,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:33,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-13T18:13:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-13T18:13:33,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-13T18:13:33,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 990 msec 2024-11-13T18:13:33,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 996 msec 2024-11-13T18:13:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:33,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:13:33,667 INFO [Thread-1198 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1198. 2024-11-13T18:13:33,667 INFO [Thread-1192 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1192. 2024-11-13T18:13:33,667 DEBUG [Thread-1198 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:33,667 DEBUG [Thread-1192 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:33,667 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,667 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:33,667 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:33,667 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:33,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:33,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:33,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/20d13d3485e04830ab43a3fac3d4cac9 is 50, key is test_row_0/A:col10/1731521613324/Put/seqid=0 2024-11-13T18:13:33,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742195_1371 (size=12301) 2024-11-13T18:13:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T18:13:33,744 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:34,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/20d13d3485e04830ab43a3fac3d4cac9 2024-11-13T18:13:34,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/3228fdf5eb1345fbac145eb599687942 is 50, key is test_row_0/B:col10/1731521613324/Put/seqid=0 2024-11-13T18:13:34,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742196_1372 (size=12301) 2024-11-13T18:13:34,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/3228fdf5eb1345fbac145eb599687942 2024-11-13T18:13:34,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/949d0deb1ddc41929fec48f5ac97dab2 is 50, key is test_row_0/C:col10/1731521613324/Put/seqid=0 2024-11-13T18:13:34,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742197_1373 (size=12301) 2024-11-13T18:13:34,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/949d0deb1ddc41929fec48f5ac97dab2 2024-11-13T18:13:34,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/20d13d3485e04830ab43a3fac3d4cac9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/20d13d3485e04830ab43a3fac3d4cac9 2024-11-13T18:13:34,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/20d13d3485e04830ab43a3fac3d4cac9, entries=150, sequenceid=616, filesize=12.0 K 2024-11-13T18:13:34,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/3228fdf5eb1345fbac145eb599687942 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/3228fdf5eb1345fbac145eb599687942 2024-11-13T18:13:34,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/3228fdf5eb1345fbac145eb599687942, entries=150, sequenceid=616, filesize=12.0 K 2024-11-13T18:13:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/949d0deb1ddc41929fec48f5ac97dab2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/949d0deb1ddc41929fec48f5ac97dab2 2024-11-13T18:13:34,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/949d0deb1ddc41929fec48f5ac97dab2, entries=150, sequenceid=616, filesize=12.0 K 2024-11-13T18:13:34,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 5c8c2fb53a8f19068c501d6951c56b68 in 467ms, sequenceid=616, compaction requested=true 2024-11-13T18:13:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c8c2fb53a8f19068c501d6951c56b68:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:34,134 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:34,134 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:34,135 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:34,135 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:34,135 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/B is initiating minor compaction (all files) 2024-11-13T18:13:34,135 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/A is initiating minor compaction (all files) 2024-11-13T18:13:34,135 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/B in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:34,135 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/A in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:34,135 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d8426a0c17b546f0a62f2a2d6820fba3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0c8d4c18f8ff4b8c97f5a374729eb72e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/20d13d3485e04830ab43a3fac3d4cac9] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=37.2 K 2024-11-13T18:13:34,135 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/fdb3ccad3c2b4db284352437c8e03dbe, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/67714b02842642aaae5362c2bdedf15f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/3228fdf5eb1345fbac145eb599687942] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=37.2 K 2024-11-13T18:13:34,136 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8426a0c17b546f0a62f2a2d6820fba3, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731521612578 2024-11-13T18:13:34,136 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting fdb3ccad3c2b4db284352437c8e03dbe, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731521612578 2024-11-13T18:13:34,136 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8d4c18f8ff4b8c97f5a374729eb72e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1731521612659 2024-11-13T18:13:34,136 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 67714b02842642aaae5362c2bdedf15f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1731521612659 2024-11-13T18:13:34,136 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20d13d3485e04830ab43a3fac3d4cac9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731521613306 2024-11-13T18:13:34,136 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 3228fdf5eb1345fbac145eb599687942, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731521613306 2024-11-13T18:13:34,146 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:34,147 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/fb82062ab8b64df893e1b8142789b997 is 50, key is test_row_0/B:col10/1731521613324/Put/seqid=0 2024-11-13T18:13:34,147 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#A#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:34,148 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/c6aa24a267e94cf88c40195e23627af3 is 50, key is test_row_0/A:col10/1731521613324/Put/seqid=0 2024-11-13T18:13:34,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742198_1374 (size=13595) 2024-11-13T18:13:34,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742199_1375 (size=13595) 2024-11-13T18:13:34,326 INFO [Thread-1194 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1194. 2024-11-13T18:13:34,326 DEBUG [Thread-1194 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:34,326 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:34,326 INFO [Thread-1196 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1196. 2024-11-13T18:13:34,326 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:34,326 DEBUG [Thread-1196 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:34,326 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:34,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:34,326 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:34,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:34,560 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/fb82062ab8b64df893e1b8142789b997 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/fb82062ab8b64df893e1b8142789b997 2024-11-13T18:13:34,562 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/c6aa24a267e94cf88c40195e23627af3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c6aa24a267e94cf88c40195e23627af3 2024-11-13T18:13:34,566 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/B of 5c8c2fb53a8f19068c501d6951c56b68 into fb82062ab8b64df893e1b8142789b997(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:34,566 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:34,566 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/B, priority=13, startTime=1731521614134; duration=0sec 2024-11-13T18:13:34,567 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:34,567 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:B 2024-11-13T18:13:34,567 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:34,569 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/A of 5c8c2fb53a8f19068c501d6951c56b68 into c6aa24a267e94cf88c40195e23627af3(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:34,569 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:34,569 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/A, priority=13, startTime=1731521614134; duration=0sec 2024-11-13T18:13:34,570 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:34,570 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:A 2024-11-13T18:13:34,570 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:34,570 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 5c8c2fb53a8f19068c501d6951c56b68/C is initiating minor compaction (all files) 2024-11-13T18:13:34,571 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c8c2fb53a8f19068c501d6951c56b68/C in TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:34,571 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b048535b9e37402ab6dae50cd4410fd3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d2cf9dd7b8b64a4fb6b7a7a9a6e5f8f8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/949d0deb1ddc41929fec48f5ac97dab2] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp, totalSize=37.2 K 2024-11-13T18:13:34,571 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b048535b9e37402ab6dae50cd4410fd3, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=568, earliestPutTs=1731521612578 2024-11-13T18:13:34,571 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d2cf9dd7b8b64a4fb6b7a7a9a6e5f8f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1731521612659 2024-11-13T18:13:34,572 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 949d0deb1ddc41929fec48f5ac97dab2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731521613306 2024-11-13T18:13:34,599 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c8c2fb53a8f19068c501d6951c56b68#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:34,599 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/de6942c6d00144caa677ebb958d98d6e is 50, key is test_row_0/C:col10/1731521613324/Put/seqid=0 2024-11-13T18:13:34,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742200_1376 (size=13595) 2024-11-13T18:13:34,649 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/de6942c6d00144caa677ebb958d98d6e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/de6942c6d00144caa677ebb958d98d6e 2024-11-13T18:13:34,657 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c8c2fb53a8f19068c501d6951c56b68/C of 5c8c2fb53a8f19068c501d6951c56b68 into de6942c6d00144caa677ebb958d98d6e(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:34,658 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c8c2fb53a8f19068c501d6951c56b68: 2024-11-13T18:13:34,658 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., storeName=5c8c2fb53a8f19068c501d6951c56b68/C, priority=13, startTime=1731521614134; duration=0sec 2024-11-13T18:13:34,658 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:34,658 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c8c2fb53a8f19068c501d6951c56b68:C 2024-11-13T18:13:34,984 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:13:42,026 INFO [Thread-1200 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1200. 2024-11-13T18:13:42,026 DEBUG [Thread-1200 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:13:42,026 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:42,026 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:42,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 150 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2270 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2219 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2298 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2273 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2228 2024-11-13T18:13:42,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:13:42,027 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:13:42,027 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:13:42,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:42,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:42,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:13:42,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:13:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:13:42,031 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521622031"}]},"ts":"1731521622031"} 2024-11-13T18:13:42,032 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:13:42,032 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:13:42,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:13:42,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c8c2fb53a8f19068c501d6951c56b68, UNASSIGN}] 2024-11-13T18:13:42,035 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c8c2fb53a8f19068c501d6951c56b68, UNASSIGN 2024-11-13T18:13:42,035 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=5c8c2fb53a8f19068c501d6951c56b68, regionState=CLOSING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:42,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c8c2fb53a8f19068c501d6951c56b68, UNASSIGN because future has completed 2024-11-13T18:13:42,037 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:13:42,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:13:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:13:42,190 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 5c8c2fb53a8f19068c501d6951c56b68, disabling compactions & flushes 2024-11-13T18:13:42,190 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. after waiting 0 ms 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:42,190 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 5c8c2fb53a8f19068c501d6951c56b68 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=A 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=B 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c8c2fb53a8f19068c501d6951c56b68, store=C 2024-11-13T18:13:42,190 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:42,195 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/fdd735b1cf154bf9947d1d3bd1c9769f is 50, key is test_row_0/A:col10/1731521622024/Put/seqid=0 2024-11-13T18:13:42,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742201_1377 (size=9857) 2024-11-13T18:13:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:13:42,599 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/fdd735b1cf154bf9947d1d3bd1c9769f 2024-11-13T18:13:42,605 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f0406e9abe7a49bba60dcbf3ab2819a4 is 50, key is test_row_0/B:col10/1731521622024/Put/seqid=0 2024-11-13T18:13:42,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742202_1378 (size=9857) 2024-11-13T18:13:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:13:43,009 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f0406e9abe7a49bba60dcbf3ab2819a4 2024-11-13T18:13:43,016 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/be7367335e7145769d50340776c4e636 is 50, key is test_row_0/C:col10/1731521622024/Put/seqid=0 2024-11-13T18:13:43,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742203_1379 (size=9857) 2024-11-13T18:13:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:13:43,422 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/be7367335e7145769d50340776c4e636 2024-11-13T18:13:43,426 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/A/fdd735b1cf154bf9947d1d3bd1c9769f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/fdd735b1cf154bf9947d1d3bd1c9769f 2024-11-13T18:13:43,430 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/fdd735b1cf154bf9947d1d3bd1c9769f, entries=100, sequenceid=625, filesize=9.6 K 2024-11-13T18:13:43,431 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/B/f0406e9abe7a49bba60dcbf3ab2819a4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f0406e9abe7a49bba60dcbf3ab2819a4 2024-11-13T18:13:43,435 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f0406e9abe7a49bba60dcbf3ab2819a4, entries=100, sequenceid=625, filesize=9.6 K 2024-11-13T18:13:43,436 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/.tmp/C/be7367335e7145769d50340776c4e636 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/be7367335e7145769d50340776c4e636 2024-11-13T18:13:43,440 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/be7367335e7145769d50340776c4e636, entries=100, sequenceid=625, filesize=9.6 K 2024-11-13T18:13:43,441 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 5c8c2fb53a8f19068c501d6951c56b68 in 1251ms, sequenceid=625, compaction requested=false 2024-11-13T18:13:43,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c1fb31f207d243deb9b3d1e607bac5ed, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/a59178724dc3484bbb75833409fe571e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0eaf1abaa6214e329f61d1856ba2d6df, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6403e30f04804dcab0640979cafd0a8d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/98f2a2cb4a5647bba78c444915c9a4c2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1839a745aa4949d6b26e1be5be2206f1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/281ea65017754a929bb84acf8c30996d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/650c434ffc12479fbe1f5f1d50611b21, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1225a14fd7424c018403dafc90209bbe, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/99f46b5783f84d229f7ade93b00a8ff3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e4279a1164304a13a64a5811c872ff02, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/fdf3dd2d10c541ecb69ce5a27cd4d288, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f5690cdaa7224797b14e4b6520bf3ec7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/89e26d57f51b42e298a828974112437e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/48530d3573774907a619aafde6ce6a7c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d095cf21b421409ab1b808fd8798d1cf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/38a455c3434f46a9a39fba799f971fd0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6a86f503405b4fa5b7786b0579c3936a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/7a6a147c96fe46f1a85869cc00f9faa0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/088d8f61324c44b187724034ad79db8d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9631a70204894203bcb4d752417fb3db, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4b5e3bd3d38b480f9f0dbe8125295e30, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9eaf7e77c0684cc0bb9ca5585a933fef, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/cca2679a27154f27aebc9857f176cd2b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/54b24cc5fd6d417ea3cbe89987a2f23c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e6753fc119a8462ba7be3a97a91a3938, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/91d5663e2e7f4a74a68d475130d0fd2c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1bf407e00acf48398c6593778e866307, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/577585accb4643cbb92cc4460e7b65c2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4861b5f884014434b67b5e0b5e8d83d3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/808393b7a9264105907518dc7440a2b3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/2713af87524f4a9cb3731eabc8926609, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/439f84f1c90f428aa4791b3e5afbdbc9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f9cff9ada3fc46fa91c87b7f2b2e4e14, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c84f53d0d35744988e530376eff63457, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d8426a0c17b546f0a62f2a2d6820fba3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0c8d4c18f8ff4b8c97f5a374729eb72e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/20d13d3485e04830ab43a3fac3d4cac9] to archive 2024-11-13T18:13:43,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:13:43,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c1fb31f207d243deb9b3d1e607bac5ed to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c1fb31f207d243deb9b3d1e607bac5ed 2024-11-13T18:13:43,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/a59178724dc3484bbb75833409fe571e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/a59178724dc3484bbb75833409fe571e 2024-11-13T18:13:43,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0eaf1abaa6214e329f61d1856ba2d6df to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0eaf1abaa6214e329f61d1856ba2d6df 2024-11-13T18:13:43,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6403e30f04804dcab0640979cafd0a8d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6403e30f04804dcab0640979cafd0a8d 2024-11-13T18:13:43,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/98f2a2cb4a5647bba78c444915c9a4c2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/98f2a2cb4a5647bba78c444915c9a4c2 2024-11-13T18:13:43,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1839a745aa4949d6b26e1be5be2206f1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1839a745aa4949d6b26e1be5be2206f1 2024-11-13T18:13:43,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/281ea65017754a929bb84acf8c30996d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/281ea65017754a929bb84acf8c30996d 2024-11-13T18:13:43,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/650c434ffc12479fbe1f5f1d50611b21 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/650c434ffc12479fbe1f5f1d50611b21 2024-11-13T18:13:43,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1225a14fd7424c018403dafc90209bbe to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1225a14fd7424c018403dafc90209bbe 2024-11-13T18:13:43,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/99f46b5783f84d229f7ade93b00a8ff3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/99f46b5783f84d229f7ade93b00a8ff3 2024-11-13T18:13:43,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e4279a1164304a13a64a5811c872ff02 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e4279a1164304a13a64a5811c872ff02 2024-11-13T18:13:43,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/fdf3dd2d10c541ecb69ce5a27cd4d288 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/fdf3dd2d10c541ecb69ce5a27cd4d288 2024-11-13T18:13:43,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f5690cdaa7224797b14e4b6520bf3ec7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f5690cdaa7224797b14e4b6520bf3ec7 2024-11-13T18:13:43,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/89e26d57f51b42e298a828974112437e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/89e26d57f51b42e298a828974112437e 2024-11-13T18:13:43,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/48530d3573774907a619aafde6ce6a7c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/48530d3573774907a619aafde6ce6a7c 2024-11-13T18:13:43,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d095cf21b421409ab1b808fd8798d1cf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d095cf21b421409ab1b808fd8798d1cf 2024-11-13T18:13:43,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/38a455c3434f46a9a39fba799f971fd0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/38a455c3434f46a9a39fba799f971fd0 2024-11-13T18:13:43,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6a86f503405b4fa5b7786b0579c3936a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/6a86f503405b4fa5b7786b0579c3936a 2024-11-13T18:13:43,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/7a6a147c96fe46f1a85869cc00f9faa0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/7a6a147c96fe46f1a85869cc00f9faa0 2024-11-13T18:13:43,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/088d8f61324c44b187724034ad79db8d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/088d8f61324c44b187724034ad79db8d 2024-11-13T18:13:43,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9631a70204894203bcb4d752417fb3db to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9631a70204894203bcb4d752417fb3db 2024-11-13T18:13:43,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4b5e3bd3d38b480f9f0dbe8125295e30 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4b5e3bd3d38b480f9f0dbe8125295e30 2024-11-13T18:13:43,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9eaf7e77c0684cc0bb9ca5585a933fef to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/9eaf7e77c0684cc0bb9ca5585a933fef 2024-11-13T18:13:43,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/cca2679a27154f27aebc9857f176cd2b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/cca2679a27154f27aebc9857f176cd2b 2024-11-13T18:13:43,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/54b24cc5fd6d417ea3cbe89987a2f23c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/54b24cc5fd6d417ea3cbe89987a2f23c 2024-11-13T18:13:43,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e6753fc119a8462ba7be3a97a91a3938 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/e6753fc119a8462ba7be3a97a91a3938 2024-11-13T18:13:43,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/91d5663e2e7f4a74a68d475130d0fd2c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/91d5663e2e7f4a74a68d475130d0fd2c 2024-11-13T18:13:43,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1bf407e00acf48398c6593778e866307 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/1bf407e00acf48398c6593778e866307 2024-11-13T18:13:43,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/577585accb4643cbb92cc4460e7b65c2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/577585accb4643cbb92cc4460e7b65c2 2024-11-13T18:13:43,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4861b5f884014434b67b5e0b5e8d83d3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/4861b5f884014434b67b5e0b5e8d83d3 2024-11-13T18:13:43,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/808393b7a9264105907518dc7440a2b3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/808393b7a9264105907518dc7440a2b3 2024-11-13T18:13:43,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/2713af87524f4a9cb3731eabc8926609 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/2713af87524f4a9cb3731eabc8926609 2024-11-13T18:13:43,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/439f84f1c90f428aa4791b3e5afbdbc9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/439f84f1c90f428aa4791b3e5afbdbc9 2024-11-13T18:13:43,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f9cff9ada3fc46fa91c87b7f2b2e4e14 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/f9cff9ada3fc46fa91c87b7f2b2e4e14 2024-11-13T18:13:43,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c84f53d0d35744988e530376eff63457 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c84f53d0d35744988e530376eff63457 2024-11-13T18:13:43,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d8426a0c17b546f0a62f2a2d6820fba3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/d8426a0c17b546f0a62f2a2d6820fba3 2024-11-13T18:13:43,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0c8d4c18f8ff4b8c97f5a374729eb72e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/0c8d4c18f8ff4b8c97f5a374729eb72e 2024-11-13T18:13:43,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/20d13d3485e04830ab43a3fac3d4cac9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/20d13d3485e04830ab43a3fac3d4cac9 2024-11-13T18:13:43,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/5bc8a41f643e4248820611f5306c56e6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e1f1e2616dd64760ac4dfcf1216ad4a1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/00206d648b594de59098cd1c8b51fdf3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/a1fda2e3ba0f4ce59bbac189769a5b68, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f696c3283a3c4a6d8e696397e265331b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/59ce033e73cc4c6491840fdc26ad0961, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/794ab2da1e7d41aab3863992b599ced2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e51006c168634843b91d1b0586407136, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/52cda14cffe644b3a00b562f29f4b6c7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/dba5d51785854eada9cf4c4c8eadda07, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/c9cd7cf411bf418b93f53f8ac1600d92, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7826e36e52744ffeb3f62f114c85e100, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/1f60a5f0fff14d40b14a6a94c8ddc153, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6fd7bc6a81304c7f8327f5e6a14ca782, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/89db2d89a9734ab6a490f9eacc377ed7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f5850f3eb01a489080812834455ccea2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/d0b21e6f90734a1eba9dd901f4efc140, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8c456227ca6848cc847cf3ed53747815, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/73cda3a35ff546a19d54a0264a1fa775, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/709686fc3d2f4acb9ee9d46154bdfd21, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6f3fa3fd45174ad7bc711751e3d7c9ff, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6e9753f2825d44f4ae9baadeb9993b62, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/edd81cb0428f4ea4ace07dbdd765874b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/aab067557dcc4698a480f9d0bece1570, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/de2ef76bf20a474880cf12cd3cf6d499, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7f9d742e4a1d4d2096bf3d1a0a90b62d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f2290edeb5ef4c9fb2af1a6c7b17e7de, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f81617c518654eb4a095c01a8a5d980d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8753023cd27a4762815ebc8fa0d0ef67, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e6766d27fe554522a0bdb6030acc73a4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/bb62b171d4db4d1482bd34f6c18854e3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/9a166fe345b145e782a2aa4b3fb9b18f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/4a97523386774a65a1dc7c620d24b8b1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/14e03172455a428c8daa1b86ab7da42c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/fdb3ccad3c2b4db284352437c8e03dbe, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6bca145f42594a06af264d687931b925, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/67714b02842642aaae5362c2bdedf15f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/3228fdf5eb1345fbac145eb599687942] to archive 2024-11-13T18:13:43,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:13:43,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/5bc8a41f643e4248820611f5306c56e6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/5bc8a41f643e4248820611f5306c56e6 2024-11-13T18:13:43,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e1f1e2616dd64760ac4dfcf1216ad4a1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e1f1e2616dd64760ac4dfcf1216ad4a1 2024-11-13T18:13:43,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/00206d648b594de59098cd1c8b51fdf3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/00206d648b594de59098cd1c8b51fdf3 2024-11-13T18:13:43,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/a1fda2e3ba0f4ce59bbac189769a5b68 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/a1fda2e3ba0f4ce59bbac189769a5b68 2024-11-13T18:13:43,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f696c3283a3c4a6d8e696397e265331b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f696c3283a3c4a6d8e696397e265331b 2024-11-13T18:13:43,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/59ce033e73cc4c6491840fdc26ad0961 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/59ce033e73cc4c6491840fdc26ad0961 2024-11-13T18:13:43,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/794ab2da1e7d41aab3863992b599ced2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/794ab2da1e7d41aab3863992b599ced2 2024-11-13T18:13:43,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e51006c168634843b91d1b0586407136 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e51006c168634843b91d1b0586407136 2024-11-13T18:13:43,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/52cda14cffe644b3a00b562f29f4b6c7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/52cda14cffe644b3a00b562f29f4b6c7 2024-11-13T18:13:43,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/dba5d51785854eada9cf4c4c8eadda07 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/dba5d51785854eada9cf4c4c8eadda07 2024-11-13T18:13:43,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/c9cd7cf411bf418b93f53f8ac1600d92 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/c9cd7cf411bf418b93f53f8ac1600d92 2024-11-13T18:13:43,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7826e36e52744ffeb3f62f114c85e100 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7826e36e52744ffeb3f62f114c85e100 2024-11-13T18:13:43,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/1f60a5f0fff14d40b14a6a94c8ddc153 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/1f60a5f0fff14d40b14a6a94c8ddc153 2024-11-13T18:13:43,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6fd7bc6a81304c7f8327f5e6a14ca782 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6fd7bc6a81304c7f8327f5e6a14ca782 2024-11-13T18:13:43,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/89db2d89a9734ab6a490f9eacc377ed7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/89db2d89a9734ab6a490f9eacc377ed7 2024-11-13T18:13:43,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f5850f3eb01a489080812834455ccea2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f5850f3eb01a489080812834455ccea2 2024-11-13T18:13:43,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/d0b21e6f90734a1eba9dd901f4efc140 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/d0b21e6f90734a1eba9dd901f4efc140 2024-11-13T18:13:43,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8c456227ca6848cc847cf3ed53747815 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8c456227ca6848cc847cf3ed53747815 2024-11-13T18:13:43,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/73cda3a35ff546a19d54a0264a1fa775 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/73cda3a35ff546a19d54a0264a1fa775 2024-11-13T18:13:43,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/709686fc3d2f4acb9ee9d46154bdfd21 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/709686fc3d2f4acb9ee9d46154bdfd21 2024-11-13T18:13:43,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6f3fa3fd45174ad7bc711751e3d7c9ff to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6f3fa3fd45174ad7bc711751e3d7c9ff 2024-11-13T18:13:43,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6e9753f2825d44f4ae9baadeb9993b62 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6e9753f2825d44f4ae9baadeb9993b62 2024-11-13T18:13:43,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/edd81cb0428f4ea4ace07dbdd765874b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/edd81cb0428f4ea4ace07dbdd765874b 2024-11-13T18:13:43,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/aab067557dcc4698a480f9d0bece1570 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/aab067557dcc4698a480f9d0bece1570 2024-11-13T18:13:43,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/de2ef76bf20a474880cf12cd3cf6d499 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/de2ef76bf20a474880cf12cd3cf6d499 2024-11-13T18:13:43,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7f9d742e4a1d4d2096bf3d1a0a90b62d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/7f9d742e4a1d4d2096bf3d1a0a90b62d 2024-11-13T18:13:43,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f2290edeb5ef4c9fb2af1a6c7b17e7de to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f2290edeb5ef4c9fb2af1a6c7b17e7de 2024-11-13T18:13:43,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f81617c518654eb4a095c01a8a5d980d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f81617c518654eb4a095c01a8a5d980d 2024-11-13T18:13:43,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8753023cd27a4762815ebc8fa0d0ef67 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/8753023cd27a4762815ebc8fa0d0ef67 2024-11-13T18:13:43,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e6766d27fe554522a0bdb6030acc73a4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/e6766d27fe554522a0bdb6030acc73a4 2024-11-13T18:13:43,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/bb62b171d4db4d1482bd34f6c18854e3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/bb62b171d4db4d1482bd34f6c18854e3 2024-11-13T18:13:43,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/9a166fe345b145e782a2aa4b3fb9b18f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/9a166fe345b145e782a2aa4b3fb9b18f 2024-11-13T18:13:43,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/4a97523386774a65a1dc7c620d24b8b1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/4a97523386774a65a1dc7c620d24b8b1 2024-11-13T18:13:43,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/14e03172455a428c8daa1b86ab7da42c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/14e03172455a428c8daa1b86ab7da42c 2024-11-13T18:13:43,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/fdb3ccad3c2b4db284352437c8e03dbe to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/fdb3ccad3c2b4db284352437c8e03dbe 2024-11-13T18:13:43,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6bca145f42594a06af264d687931b925 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/6bca145f42594a06af264d687931b925 2024-11-13T18:13:43,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/67714b02842642aaae5362c2bdedf15f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/67714b02842642aaae5362c2bdedf15f 2024-11-13T18:13:43,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/3228fdf5eb1345fbac145eb599687942 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/3228fdf5eb1345fbac145eb599687942 2024-11-13T18:13:43,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/000750ab0c7142cc91275a81acb10f75, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/32e14c3043bd45d29a2795d269b7abeb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8b799188969c4fc5986d0b99099c13eb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d4b41d18d6e74f1689a090ca92036530, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da7e0e16478847f483e70f6496667059, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d1722c2971834959a7c8b794ff032da8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/6781a7b94e5940d3872ea2bb977a1644, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5f57fec9b5d497d9ea0912c4d8f5802, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d58b3f90d56145a58d6457656896dfe7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/c69b950d057541e2af6854b190b4bb8a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/896c6704f83e4a3fad39d9e131e7f9ed, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b9e6789dce0146938b94f031cf31f32b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b61815199c014bd5951ae2e6843106ec, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/0133a20649f54e1587938ac639392758, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b7976af3804e456ea7ddd42d646f63c9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da943ba3e98c424699a33a46669e3a61, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/ec656bab74d54136a87ebf59f9df0f78, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/efb0c8905fbf4de98b77e45febfa5e0f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5e9cc1a47c742b4a92d057b61495869, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/34fd725383a44e44a1f2853d6b9967a8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/858c2fe0854f46e3919e0fc6cfe77431, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/e91c1c42fed0421cbb3f90ff6bb3e038, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d47eaa8d5d2740f58a76eb712cb73eec, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/57f314cb7ae64fb5bc6cde1fa0ae09e5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a9ea4a7cc12b4188ba6e19a1ccf1c0d6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/0b5765bf69d3400d885ba95815c3adac, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/41465f3125a4454cb3fc1e09c4be663a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8462825c7cd74824bf553bcd16222f8a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/482afb75a28a4f6cbb1bc91bf1e97d65, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/5332274a2f914757b8e402e5c1850d53, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/4f32c072e848483f8354cda0d1d4ab4b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/cecfb098c2bf4b24864379542f1dbc30, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/23de1dd4a4c244a3a00eb2791a695e4e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/58eb3f5bd3604ccea2e0a1117315ac39, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b048535b9e37402ab6dae50cd4410fd3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a48fe1a41bfc4336b773ddb3db8f8365, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d2cf9dd7b8b64a4fb6b7a7a9a6e5f8f8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/949d0deb1ddc41929fec48f5ac97dab2] to archive 2024-11-13T18:13:43,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:13:43,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/000750ab0c7142cc91275a81acb10f75 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/000750ab0c7142cc91275a81acb10f75 2024-11-13T18:13:43,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/32e14c3043bd45d29a2795d269b7abeb to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/32e14c3043bd45d29a2795d269b7abeb 2024-11-13T18:13:43,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8b799188969c4fc5986d0b99099c13eb to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8b799188969c4fc5986d0b99099c13eb 2024-11-13T18:13:43,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d4b41d18d6e74f1689a090ca92036530 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d4b41d18d6e74f1689a090ca92036530 2024-11-13T18:13:43,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da7e0e16478847f483e70f6496667059 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da7e0e16478847f483e70f6496667059 2024-11-13T18:13:43,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d1722c2971834959a7c8b794ff032da8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d1722c2971834959a7c8b794ff032da8 2024-11-13T18:13:43,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/6781a7b94e5940d3872ea2bb977a1644 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/6781a7b94e5940d3872ea2bb977a1644 2024-11-13T18:13:43,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5f57fec9b5d497d9ea0912c4d8f5802 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5f57fec9b5d497d9ea0912c4d8f5802 2024-11-13T18:13:43,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d58b3f90d56145a58d6457656896dfe7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d58b3f90d56145a58d6457656896dfe7 2024-11-13T18:13:43,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/c69b950d057541e2af6854b190b4bb8a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/c69b950d057541e2af6854b190b4bb8a 2024-11-13T18:13:43,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/896c6704f83e4a3fad39d9e131e7f9ed to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/896c6704f83e4a3fad39d9e131e7f9ed 2024-11-13T18:13:43,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b9e6789dce0146938b94f031cf31f32b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b9e6789dce0146938b94f031cf31f32b 2024-11-13T18:13:43,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b61815199c014bd5951ae2e6843106ec to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b61815199c014bd5951ae2e6843106ec 2024-11-13T18:13:43,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/0133a20649f54e1587938ac639392758 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/0133a20649f54e1587938ac639392758 2024-11-13T18:13:43,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b7976af3804e456ea7ddd42d646f63c9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b7976af3804e456ea7ddd42d646f63c9 2024-11-13T18:13:43,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da943ba3e98c424699a33a46669e3a61 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/da943ba3e98c424699a33a46669e3a61 2024-11-13T18:13:43,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/ec656bab74d54136a87ebf59f9df0f78 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/ec656bab74d54136a87ebf59f9df0f78 2024-11-13T18:13:43,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/efb0c8905fbf4de98b77e45febfa5e0f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/efb0c8905fbf4de98b77e45febfa5e0f 2024-11-13T18:13:43,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5e9cc1a47c742b4a92d057b61495869 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a5e9cc1a47c742b4a92d057b61495869 2024-11-13T18:13:43,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/34fd725383a44e44a1f2853d6b9967a8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/34fd725383a44e44a1f2853d6b9967a8 2024-11-13T18:13:43,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/858c2fe0854f46e3919e0fc6cfe77431 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/858c2fe0854f46e3919e0fc6cfe77431 2024-11-13T18:13:43,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/e91c1c42fed0421cbb3f90ff6bb3e038 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/e91c1c42fed0421cbb3f90ff6bb3e038 2024-11-13T18:13:43,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d47eaa8d5d2740f58a76eb712cb73eec to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d47eaa8d5d2740f58a76eb712cb73eec 2024-11-13T18:13:43,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/57f314cb7ae64fb5bc6cde1fa0ae09e5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/57f314cb7ae64fb5bc6cde1fa0ae09e5 2024-11-13T18:13:43,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a9ea4a7cc12b4188ba6e19a1ccf1c0d6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a9ea4a7cc12b4188ba6e19a1ccf1c0d6 2024-11-13T18:13:43,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/0b5765bf69d3400d885ba95815c3adac to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/0b5765bf69d3400d885ba95815c3adac 2024-11-13T18:13:43,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/41465f3125a4454cb3fc1e09c4be663a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/41465f3125a4454cb3fc1e09c4be663a 2024-11-13T18:13:43,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8462825c7cd74824bf553bcd16222f8a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/8462825c7cd74824bf553bcd16222f8a 2024-11-13T18:13:43,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/482afb75a28a4f6cbb1bc91bf1e97d65 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/482afb75a28a4f6cbb1bc91bf1e97d65 2024-11-13T18:13:43,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/5332274a2f914757b8e402e5c1850d53 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/5332274a2f914757b8e402e5c1850d53 2024-11-13T18:13:43,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/4f32c072e848483f8354cda0d1d4ab4b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/4f32c072e848483f8354cda0d1d4ab4b 2024-11-13T18:13:43,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/cecfb098c2bf4b24864379542f1dbc30 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/cecfb098c2bf4b24864379542f1dbc30 2024-11-13T18:13:43,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/23de1dd4a4c244a3a00eb2791a695e4e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/23de1dd4a4c244a3a00eb2791a695e4e 2024-11-13T18:13:43,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/58eb3f5bd3604ccea2e0a1117315ac39 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/58eb3f5bd3604ccea2e0a1117315ac39 2024-11-13T18:13:43,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b048535b9e37402ab6dae50cd4410fd3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/b048535b9e37402ab6dae50cd4410fd3 2024-11-13T18:13:43,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a48fe1a41bfc4336b773ddb3db8f8365 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/a48fe1a41bfc4336b773ddb3db8f8365 2024-11-13T18:13:43,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d2cf9dd7b8b64a4fb6b7a7a9a6e5f8f8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/d2cf9dd7b8b64a4fb6b7a7a9a6e5f8f8 2024-11-13T18:13:43,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/949d0deb1ddc41929fec48f5ac97dab2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/949d0deb1ddc41929fec48f5ac97dab2 2024-11-13T18:13:43,570 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/recovered.edits/628.seqid, newMaxSeqId=628, maxSeqId=1 2024-11-13T18:13:43,570 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. 2024-11-13T18:13:43,570 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 5c8c2fb53a8f19068c501d6951c56b68: Waiting for close lock at 1731521622190Running coprocessor pre-close hooks at 1731521622190Disabling compacts and flushes for region at 1731521622190Disabling writes for close at 1731521622190Obtaining lock to block concurrent updates at 1731521622190Preparing flush snapshotting stores in 5c8c2fb53a8f19068c501d6951c56b68 at 1731521622190Finished memstore snapshotting TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731521622190Flushing stores of TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68. at 1731521622191 (+1 ms)Flushing 5c8c2fb53a8f19068c501d6951c56b68/A: creating writer at 1731521622191Flushing 5c8c2fb53a8f19068c501d6951c56b68/A: appending metadata at 1731521622194 (+3 ms)Flushing 5c8c2fb53a8f19068c501d6951c56b68/A: closing flushed file at 1731521622194Flushing 5c8c2fb53a8f19068c501d6951c56b68/B: creating writer at 1731521622603 (+409 ms)Flushing 5c8c2fb53a8f19068c501d6951c56b68/B: appending metadata at 1731521622605 (+2 ms)Flushing 5c8c2fb53a8f19068c501d6951c56b68/B: closing flushed file at 1731521622605Flushing 5c8c2fb53a8f19068c501d6951c56b68/C: creating writer at 1731521623013 (+408 ms)Flushing 5c8c2fb53a8f19068c501d6951c56b68/C: appending metadata at 1731521623015 (+2 ms)Flushing 5c8c2fb53a8f19068c501d6951c56b68/C: closing flushed file at 1731521623015Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70f6f385: reopening flushed file at 1731521623426 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3626611c: reopening flushed file at 1731521623430 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55037528: reopening flushed file at 1731521623435 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 5c8c2fb53a8f19068c501d6951c56b68 in 1251ms, sequenceid=625, compaction requested=false at 1731521623441 (+6 ms)Writing region close event to WAL at 1731521623567 (+126 ms)Running coprocessor post-close hooks at 1731521623570 (+3 ms)Closed at 1731521623570 2024-11-13T18:13:43,572 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:43,572 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=5c8c2fb53a8f19068c501d6951c56b68, regionState=CLOSED 2024-11-13T18:13:43,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:13:43,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-13T18:13:43,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 5c8c2fb53a8f19068c501d6951c56b68, server=fda96cf025dc,44483,1731521526892 in 1.5380 sec 2024-11-13T18:13:43,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-13T18:13:43,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c8c2fb53a8f19068c501d6951c56b68, UNASSIGN in 1.5420 sec 2024-11-13T18:13:43,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-13T18:13:43,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5450 sec 2024-11-13T18:13:43,581 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521623580"}]},"ts":"1731521623580"} 2024-11-13T18:13:43,582 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:13:43,582 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:13:43,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5550 sec 2024-11-13T18:13:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:13:44,175 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:44,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:13:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:44,178 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T18:13:44,179 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:44,181 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:44,183 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/recovered.edits] 2024-11-13T18:13:44,187 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c6aa24a267e94cf88c40195e23627af3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/c6aa24a267e94cf88c40195e23627af3 2024-11-13T18:13:44,188 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/fdd735b1cf154bf9947d1d3bd1c9769f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/A/fdd735b1cf154bf9947d1d3bd1c9769f 2024-11-13T18:13:44,190 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f0406e9abe7a49bba60dcbf3ab2819a4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/f0406e9abe7a49bba60dcbf3ab2819a4 2024-11-13T18:13:44,190 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/fb82062ab8b64df893e1b8142789b997 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/B/fb82062ab8b64df893e1b8142789b997 2024-11-13T18:13:44,192 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/be7367335e7145769d50340776c4e636 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/be7367335e7145769d50340776c4e636 2024-11-13T18:13:44,193 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/de6942c6d00144caa677ebb958d98d6e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/C/de6942c6d00144caa677ebb958d98d6e 2024-11-13T18:13:44,195 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/recovered.edits/628.seqid to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68/recovered.edits/628.seqid 2024-11-13T18:13:44,196 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/5c8c2fb53a8f19068c501d6951c56b68 2024-11-13T18:13:44,196 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:13:44,198 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:44,200 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:13:44,202 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:13:44,203 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:44,203 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:13:44,204 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731521624203"}]},"ts":"9223372036854775807"} 2024-11-13T18:13:44,206 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:13:44,206 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5c8c2fb53a8f19068c501d6951c56b68, NAME => 'TestAcidGuarantees,,1731521591256.5c8c2fb53a8f19068c501d6951c56b68.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:13:44,206 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:13:44,206 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731521624206"}]},"ts":"9223372036854775807"} 2024-11-13T18:13:44,208 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:13:44,208 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:44,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-13T18:13:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T18:13:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:13:44,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:44,296 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=716 (was 663) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=569 (was 809) 2024-11-13T18:13:44,305 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=716, ProcessCount=11, AvailableMemoryMB=569 2024-11-13T18:13:44,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:13:44,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:13:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:44,308 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:13:44,308 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:44,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-13T18:13:44,309 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:13:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-13T18:13:44,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742204_1380 (size=1061) 2024-11-13T18:13:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-13T18:13:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-13T18:13:44,717 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1 2024-11-13T18:13:44,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742205_1381 (size=53) 2024-11-13T18:13:44,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:13:44,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fc0f52f0b3b547094f90472a203572fc, disabling compactions & flushes 2024-11-13T18:13:44,726 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:44,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:44,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. after waiting 0 ms 2024-11-13T18:13:44,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:44,726 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:44,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fc0f52f0b3b547094f90472a203572fc: Waiting for close lock at 1731521624726Disabling compacts and flushes for region at 1731521624726Disabling writes for close at 1731521624726Writing region close event to WAL at 1731521624726Closed at 1731521624726 2024-11-13T18:13:44,727 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:13:44,727 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731521624727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521624727"}]},"ts":"1731521624727"} 2024-11-13T18:13:44,729 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:13:44,730 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:13:44,730 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521624730"}]},"ts":"1731521624730"} 2024-11-13T18:13:44,732 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:13:44,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, ASSIGN}] 2024-11-13T18:13:44,733 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, ASSIGN 2024-11-13T18:13:44,734 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, ASSIGN; state=OFFLINE, location=fda96cf025dc,44483,1731521526892; forceNewPlan=false, retain=false 2024-11-13T18:13:44,885 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=fc0f52f0b3b547094f90472a203572fc, regionState=OPENING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:44,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, ASSIGN because future has completed 2024-11-13T18:13:44,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:13:44,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-13T18:13:45,043 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,043 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:13:45,043 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,043 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:13:45,043 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,043 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,044 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,046 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:13:45,046 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc0f52f0b3b547094f90472a203572fc columnFamilyName A 2024-11-13T18:13:45,046 DEBUG [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:45,046 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(327): Store=fc0f52f0b3b547094f90472a203572fc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:13:45,047 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,047 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:13:45,048 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc0f52f0b3b547094f90472a203572fc columnFamilyName B 2024-11-13T18:13:45,048 DEBUG [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:45,048 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(327): Store=fc0f52f0b3b547094f90472a203572fc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:13:45,048 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,049 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:13:45,049 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc0f52f0b3b547094f90472a203572fc columnFamilyName C 2024-11-13T18:13:45,049 DEBUG [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:45,049 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(327): Store=fc0f52f0b3b547094f90472a203572fc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:13:45,050 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,050 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,050 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,051 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,051 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,051 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,052 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:13:45,053 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,054 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:13:45,054 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened fc0f52f0b3b547094f90472a203572fc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65897859, jitterRate=-0.01804538071155548}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:13:45,055 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,055 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for fc0f52f0b3b547094f90472a203572fc: Running coprocessor pre-open hook at 1731521625043Writing region info on filesystem at 1731521625043Initializing all the Stores at 1731521625044 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521625044Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521625044Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521625044Cleaning up temporary data from old regions at 1731521625051 (+7 ms)Running coprocessor post-open hooks at 1731521625055 (+4 ms)Region opened successfully at 1731521625055 2024-11-13T18:13:45,056 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., pid=85, masterSystemTime=1731521625039 2024-11-13T18:13:45,058 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,058 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,058 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=fc0f52f0b3b547094f90472a203572fc, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:45,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:13:45,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-13T18:13:45,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 in 174 msec 2024-11-13T18:13:45,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-13T18:13:45,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, ASSIGN in 331 msec 2024-11-13T18:13:45,066 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:13:45,066 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521625066"}]},"ts":"1731521625066"} 2024-11-13T18:13:45,068 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:13:45,068 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:13:45,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 762 msec 2024-11-13T18:13:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-13T18:13:45,444 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:45,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ccd502a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:45,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:45,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:45,446 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d9be759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:45,448 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4696f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:45,449 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:45,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:45,452 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:13:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,37165,1731521525941 2024-11-13T18:13:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@318ef17c 2024-11-13T18:13:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:13:45,454 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:13:45,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:13:45,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:13:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T18:13:45,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742206_1382 (size=1097) 2024-11-13T18:13:45,468 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T18:13:45,468 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T18:13:45,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:13:45,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, REOPEN/MOVE}] 2024-11-13T18:13:45,473 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, REOPEN/MOVE 2024-11-13T18:13:45,473 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=fc0f52f0b3b547094f90472a203572fc, regionState=CLOSING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:45,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, REOPEN/MOVE because future has completed 2024-11-13T18:13:45,475 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:13:45,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:13:45,627 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:13:45,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing fc0f52f0b3b547094f90472a203572fc, disabling compactions & flushes 2024-11-13T18:13:45,628 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. after waiting 0 ms 2024-11-13T18:13:45,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,632 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:13:45,632 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,632 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for fc0f52f0b3b547094f90472a203572fc: Waiting for close lock at 1731521625628Running coprocessor pre-close hooks at 1731521625628Disabling compacts and flushes for region at 1731521625628Disabling writes for close at 1731521625628Writing region close event to WAL at 1731521625628Running coprocessor post-close hooks at 1731521625632 (+4 ms)Closed at 1731521625632 2024-11-13T18:13:45,632 WARN [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: fc0f52f0b3b547094f90472a203572fc to self. 2024-11-13T18:13:45,634 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,635 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=fc0f52f0b3b547094f90472a203572fc, regionState=CLOSED 2024-11-13T18:13:45,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:13:45,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-13T18:13:45,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 in 162 msec 2024-11-13T18:13:45,639 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, REOPEN/MOVE; state=CLOSED, location=fda96cf025dc,44483,1731521526892; forceNewPlan=false, retain=true 2024-11-13T18:13:45,790 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=fc0f52f0b3b547094f90472a203572fc, regionState=OPENING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:45,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, REOPEN/MOVE because future has completed 2024-11-13T18:13:45,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:13:45,948 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,948 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:13:45,948 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,948 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:13:45,948 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,948 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,950 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,951 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:13:45,951 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc0f52f0b3b547094f90472a203572fc columnFamilyName A 2024-11-13T18:13:45,952 DEBUG [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:45,954 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(327): Store=fc0f52f0b3b547094f90472a203572fc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:13:45,954 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,955 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:13:45,955 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc0f52f0b3b547094f90472a203572fc columnFamilyName B 2024-11-13T18:13:45,955 DEBUG [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:45,956 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(327): Store=fc0f52f0b3b547094f90472a203572fc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:13:45,956 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,959 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:13:45,959 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc0f52f0b3b547094f90472a203572fc columnFamilyName C 2024-11-13T18:13:45,959 DEBUG [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:45,960 INFO [StoreOpener-fc0f52f0b3b547094f90472a203572fc-1 {}] regionserver.HStore(327): Store=fc0f52f0b3b547094f90472a203572fc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:13:45,960 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,960 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,960 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,962 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,964 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,964 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,965 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:13:45,966 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,968 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened fc0f52f0b3b547094f90472a203572fc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75415717, jitterRate=0.12378175556659698}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:13:45,968 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:45,969 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for fc0f52f0b3b547094f90472a203572fc: Running coprocessor pre-open hook at 1731521625948Writing region info on filesystem at 1731521625948Initializing all the Stores at 1731521625949 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521625949Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521625950 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521625950Cleaning up temporary data from old regions at 1731521625965 (+15 ms)Running coprocessor post-open hooks at 1731521625968 (+3 ms)Region opened successfully at 1731521625969 (+1 ms) 2024-11-13T18:13:45,971 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., pid=90, masterSystemTime=1731521625945 2024-11-13T18:13:45,981 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=fc0f52f0b3b547094f90472a203572fc, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:45,983 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,983 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:45,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:13:45,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-13T18:13:45,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 in 192 msec 2024-11-13T18:13:45,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-13T18:13:45,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, REOPEN/MOVE in 516 msec 2024-11-13T18:13:45,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 540 msec 2024-11-13T18:13:45,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-13T18:13:45,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-11-13T18:13:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T18:13:45,999 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:46,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7514252c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:46,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:46,008 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61029936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:46,013 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a91140f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69763936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:46,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:46,018 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed6e1c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:46,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:46,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c35846, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf8eb63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:46,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:46,040 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a20c87e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:46,042 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75678128, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@371c0a12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,063 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:46,063 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:46,065 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce5517b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:46,067 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f728ee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7183b00c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,070 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:46,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:46,070 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b2a811, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:46,072 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41012f73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da191d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:46,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:46,077 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@410693f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:46,079 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@589a6303, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51c6b1d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:46,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:46,098 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d4c822, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:46,102 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:46,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d4af0f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37999ed3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:46,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:46,104 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6418ad69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:46,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:46,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ee33d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758d494c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:46,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:46,109 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:46,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1beac921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:46,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:46,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:46,111 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:46,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c57b72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7555a6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:13:46,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:13:46,114 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:13:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:13:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:13:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f82dae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:13:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:13:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:13:46,116 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:13:46,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743963a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:13:46,125 DEBUG [Thread-1721 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:46,126 DEBUG [Thread-1721 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:46,127 DEBUG [Thread-1721 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:46,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:46,132 DEBUG [Thread-1738 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:46,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:13:46,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:46,138 DEBUG [Thread-1719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:13:46,139 DEBUG [Thread-1719 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:46,139 DEBUG [Thread-1719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:46,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:46,141 DEBUG [Thread-1734 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:46,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:13:46,143 DEBUG [Thread-1734 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:46,143 DEBUG [Thread-1734 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:46,144 DEBUG [Thread-1723 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:46,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:46,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:13:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-13T18:13:46,147 DEBUG [Thread-1730 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:46,148 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:46,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:46,152 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:46,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:46,154 DEBUG [Thread-1730 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:46,155 DEBUG [Thread-1730 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:46,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:13:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:46,158 DEBUG [Thread-1730 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:13:46,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:13:46,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:46,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:46,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:46,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:46,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:46,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:46,162 DEBUG [Thread-1727 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:46,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:46,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:13:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:13:46,167 DEBUG [Thread-1732 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:46,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:13:46,185 DEBUG [Thread-1725 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:46,189 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:13:46,199 DEBUG [Thread-1736 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:13:46,200 DEBUG [Thread-1736 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:13:46,200 DEBUG [Thread-1736 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:13:46,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:13:46,204 DEBUG [Thread-1736 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:13:46,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731521626194, max timestamp=1731521626236Num uniques 859; 2024-11-13T18:13:46,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:13:46,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731521626194, max timestamp=1731521626236Num uniques 859; 2024-11-13T18:13:46,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:13:46,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731521626194, max timestamp=1731521626236Num uniques 859; 2024-11-13T18:13:46,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:13:46,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111336ddc027663c4786883c3e47ba92cf63_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521626155/Put/seqid=0 2024-11-13T18:13:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:13:46,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:13:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:46,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742207_1383 (size=12154) 2024-11-13T18:13:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521686295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521686304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521686306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521686313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521686330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521686434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521686434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521686434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521686446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:13:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:46,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:13:46,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521686454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:13:46,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:46,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:46,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:46,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521686664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521686674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521686674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521686676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521686704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:46,718 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:46,723 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111336ddc027663c4786883c3e47ba92cf63_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111336ddc027663c4786883c3e47ba92cf63_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:46,723 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/8c0a274ed76548d0876da72b23af709c, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:46,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/8c0a274ed76548d0876da72b23af709c is 175, key is test_row_0/A:col10/1731521626155/Put/seqid=0 2024-11-13T18:13:46,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742208_1384 (size=30955) 2024-11-13T18:13:46,753 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/8c0a274ed76548d0876da72b23af709c 2024-11-13T18:13:46,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:13:46,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:46,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:46,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:46,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:46,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/dc9dc39bb00a4503b98f40495f41507a is 50, key is test_row_0/B:col10/1731521626155/Put/seqid=0 2024-11-13T18:13:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:13:46,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742209_1385 (size=12001) 2024-11-13T18:13:46,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/dc9dc39bb00a4503b98f40495f41507a 2024-11-13T18:13:46,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/ed5f6ed4d3a4451486ab8d748444499d is 50, key is test_row_0/C:col10/1731521626155/Put/seqid=0 2024-11-13T18:13:46,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742210_1386 (size=12001) 2024-11-13T18:13:46,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/ed5f6ed4d3a4451486ab8d748444499d 2024-11-13T18:13:46,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/8c0a274ed76548d0876da72b23af709c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8c0a274ed76548d0876da72b23af709c 2024-11-13T18:13:46,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8c0a274ed76548d0876da72b23af709c, entries=150, sequenceid=15, filesize=30.2 K 2024-11-13T18:13:46,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/dc9dc39bb00a4503b98f40495f41507a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/dc9dc39bb00a4503b98f40495f41507a 2024-11-13T18:13:46,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/dc9dc39bb00a4503b98f40495f41507a, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T18:13:46,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/ed5f6ed4d3a4451486ab8d748444499d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ed5f6ed4d3a4451486ab8d748444499d 2024-11-13T18:13:46,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ed5f6ed4d3a4451486ab8d748444499d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T18:13:46,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fc0f52f0b3b547094f90472a203572fc in 726ms, sequenceid=15, compaction requested=false 2024-11-13T18:13:46,884 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T18:13:46,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:46,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:13:46,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:46,943 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-13T18:13:46,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:46,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:46,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:46,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:46,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:46,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:46,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111391f96b05d0a04796b197f0ed1ebcb2b4_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521626323/Put/seqid=0 2024-11-13T18:13:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:47,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742211_1387 (size=12154) 2024-11-13T18:13:47,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:47,013 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111391f96b05d0a04796b197f0ed1ebcb2b4_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111391f96b05d0a04796b197f0ed1ebcb2b4_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:47,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0 is 175, key is test_row_0/A:col10/1731521626323/Put/seqid=0 2024-11-13T18:13:47,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521687028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521687032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521687033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742212_1388 (size=30955) 2024-11-13T18:13:47,042 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0 2024-11-13T18:13:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/b44cf2848b2c4c21adb94e15b760f896 is 50, key is test_row_0/B:col10/1731521626323/Put/seqid=0 2024-11-13T18:13:47,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521687039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742213_1389 (size=12001) 2024-11-13T18:13:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521687041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,065 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/b44cf2848b2c4c21adb94e15b760f896 2024-11-13T18:13:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/39dcc7667eef494e8b9e5558c57e7226 is 50, key is test_row_0/C:col10/1731521626323/Put/seqid=0 2024-11-13T18:13:47,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742214_1390 (size=12001) 2024-11-13T18:13:47,094 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/39dcc7667eef494e8b9e5558c57e7226 2024-11-13T18:13:47,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0 2024-11-13T18:13:47,114 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0, entries=150, sequenceid=49, filesize=30.2 K 2024-11-13T18:13:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/b44cf2848b2c4c21adb94e15b760f896 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b44cf2848b2c4c21adb94e15b760f896 2024-11-13T18:13:47,123 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b44cf2848b2c4c21adb94e15b760f896, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T18:13:47,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/39dcc7667eef494e8b9e5558c57e7226 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/39dcc7667eef494e8b9e5558c57e7226 2024-11-13T18:13:47,133 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/39dcc7667eef494e8b9e5558c57e7226, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T18:13:47,134 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for fc0f52f0b3b547094f90472a203572fc in 191ms, sequenceid=49, compaction requested=false 2024-11-13T18:13:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-13T18:13:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-13T18:13:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:47,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-13T18:13:47,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0280 sec 2024-11-13T18:13:47,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:13:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:47,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.0400 sec 2024-11-13T18:13:47,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139bea4de288444661b8339743a7579c47_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521627180/Put/seqid=0 2024-11-13T18:13:47,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742215_1391 (size=12154) 2024-11-13T18:13:47,228 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:47,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139bea4de288444661b8339743a7579c47_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139bea4de288444661b8339743a7579c47_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:47,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/39f79203dffe49468069a7c2e6b588de, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:47,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/39f79203dffe49468069a7c2e6b588de is 175, key is test_row_0/A:col10/1731521627180/Put/seqid=0 2024-11-13T18:13:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521687251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521687252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521687258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521687250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742216_1392 (size=30955) 2024-11-13T18:13:47,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521687275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:13:47,305 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:47,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-13T18:13:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:13:47,311 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:47,312 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:47,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:47,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521687384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521687384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521687384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521687384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:13:47,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521687404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:13:47,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:47,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:47,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:47,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:47,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521687605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521687605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521687605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521687605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:13:47,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:47,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:47,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:47,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:47,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:13:47,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521687629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,691 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/39f79203dffe49468069a7c2e6b588de 2024-11-13T18:13:47,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/388b886260954ae1b8173b5908a8592f is 50, key is test_row_0/B:col10/1731521627180/Put/seqid=0 2024-11-13T18:13:47,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742217_1393 (size=12001) 2024-11-13T18:13:47,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/388b886260954ae1b8173b5908a8592f 2024-11-13T18:13:47,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/74c5b0cf33bf452db8a07b6af43a0cb3 is 50, key is test_row_0/C:col10/1731521627180/Put/seqid=0 2024-11-13T18:13:47,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742218_1394 (size=12001) 2024-11-13T18:13:47,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/74c5b0cf33bf452db8a07b6af43a0cb3 2024-11-13T18:13:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/39f79203dffe49468069a7c2e6b588de as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/39f79203dffe49468069a7c2e6b588de 2024-11-13T18:13:47,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/39f79203dffe49468069a7c2e6b588de, entries=150, sequenceid=64, filesize=30.2 K 2024-11-13T18:13:47,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/388b886260954ae1b8173b5908a8592f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/388b886260954ae1b8173b5908a8592f 2024-11-13T18:13:47,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/388b886260954ae1b8173b5908a8592f, entries=150, sequenceid=64, filesize=11.7 K 2024-11-13T18:13:47,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/74c5b0cf33bf452db8a07b6af43a0cb3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74c5b0cf33bf452db8a07b6af43a0cb3 2024-11-13T18:13:47,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74c5b0cf33bf452db8a07b6af43a0cb3, entries=150, sequenceid=64, filesize=11.7 K 2024-11-13T18:13:47,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fc0f52f0b3b547094f90472a203572fc in 590ms, sequenceid=64, compaction requested=true 2024-11-13T18:13:47,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:47,774 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:47,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:47,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:47,774 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:47,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:47,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:47,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:47,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:47,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:13:47,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:47,775 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:13:47,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:47,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:47,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:47,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:47,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:47,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:47,779 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:47,779 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:47,779 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:13:47,779 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:13:47,779 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:47,779 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:47,779 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8c0a274ed76548d0876da72b23af709c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/39f79203dffe49468069a7c2e6b588de] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=90.7 K 2024-11-13T18:13:47,779 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/dc9dc39bb00a4503b98f40495f41507a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b44cf2848b2c4c21adb94e15b760f896, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/388b886260954ae1b8173b5908a8592f] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=35.2 K 2024-11-13T18:13:47,779 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:47,779 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8c0a274ed76548d0876da72b23af709c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/39f79203dffe49468069a7c2e6b588de] 2024-11-13T18:13:47,785 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting dc9dc39bb00a4503b98f40495f41507a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731521626143 2024-11-13T18:13:47,785 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c0a274ed76548d0876da72b23af709c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731521626143 2024-11-13T18:13:47,786 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b44cf2848b2c4c21adb94e15b760f896, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731521626283 2024-11-13T18:13:47,786 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc0e5438cec4c4d97c4c9db9f8e3fd0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731521626283 2024-11-13T18:13:47,786 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 388b886260954ae1b8173b5908a8592f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731521627035 2024-11-13T18:13:47,786 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39f79203dffe49468069a7c2e6b588de, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731521627035 2024-11-13T18:13:47,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c689174a00904411ade188b8f99d9eb7_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521627249/Put/seqid=0 2024-11-13T18:13:47,807 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:47,807 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/02539b7ef98242cfb4b931c074c7306c is 50, key is test_row_0/B:col10/1731521627180/Put/seqid=0 2024-11-13T18:13:47,808 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:47,817 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113325d259ab6f846bfa01b834770786d82_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:47,820 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113325d259ab6f846bfa01b834770786d82_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:47,820 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113325d259ab6f846bfa01b834770786d82_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:47,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742219_1395 (size=12154) 2024-11-13T18:13:47,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742220_1396 (size=12104) 2024-11-13T18:13:47,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:47,843 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c689174a00904411ade188b8f99d9eb7_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c689174a00904411ade188b8f99d9eb7_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:47,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742221_1397 (size=4469) 2024-11-13T18:13:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/ef31d3769b4e46169df4a9a33d201d5d, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:47,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/ef31d3769b4e46169df4a9a33d201d5d is 175, key is test_row_0/A:col10/1731521627249/Put/seqid=0 2024-11-13T18:13:47,851 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#341 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:47,852 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/efd970fd26784d7d81e54900417b3878 is 175, key is test_row_0/A:col10/1731521627180/Put/seqid=0 2024-11-13T18:13:47,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742222_1398 (size=31058) 2024-11-13T18:13:47,920 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/efd970fd26784d7d81e54900417b3878 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/efd970fd26784d7d81e54900417b3878 2024-11-13T18:13:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:47,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:47,934 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into efd970fd26784d7d81e54900417b3878(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:47,934 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:47,934 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521627774; duration=0sec 2024-11-13T18:13:47,934 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:47,934 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:13:47,934 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:13:47,935 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:47,935 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/C is initiating minor compaction (all files) 2024-11-13T18:13:47,935 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/C in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:47,935 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ed5f6ed4d3a4451486ab8d748444499d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/39dcc7667eef494e8b9e5558c57e7226, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74c5b0cf33bf452db8a07b6af43a0cb3] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=35.2 K 2024-11-13T18:13:47,936 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed5f6ed4d3a4451486ab8d748444499d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731521626143 2024-11-13T18:13:47,936 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39dcc7667eef494e8b9e5558c57e7226, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731521626283 2024-11-13T18:13:47,936 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74c5b0cf33bf452db8a07b6af43a0cb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731521627035 2024-11-13T18:13:47,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742223_1399 (size=30955) 2024-11-13T18:13:47,945 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/ef31d3769b4e46169df4a9a33d201d5d 2024-11-13T18:13:47,955 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#C#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:47,955 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/a36f51b1961e435aa51118a4283afecd is 50, key is test_row_0/C:col10/1731521627180/Put/seqid=0 2024-11-13T18:13:47,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/6382c2ebfd7343829b0dc4eafb138677 is 50, key is test_row_0/B:col10/1731521627249/Put/seqid=0 2024-11-13T18:13:47,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521687967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521687969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521687972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521687973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:47,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521687976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742225_1401 (size=12001) 2024-11-13T18:13:48,003 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/6382c2ebfd7343829b0dc4eafb138677 2024-11-13T18:13:48,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/1c319efe19b3453e9f4b08473b05997f is 50, key is test_row_0/C:col10/1731521627249/Put/seqid=0 2024-11-13T18:13:48,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742224_1400 (size=12104) 2024-11-13T18:13:48,036 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/a36f51b1961e435aa51118a4283afecd as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a36f51b1961e435aa51118a4283afecd 2024-11-13T18:13:48,056 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/C of fc0f52f0b3b547094f90472a203572fc into a36f51b1961e435aa51118a4283afecd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:48,056 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:48,056 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/C, priority=13, startTime=1731521627775; duration=0sec 2024-11-13T18:13:48,057 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:48,057 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:13:48,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742226_1402 (size=12001) 2024-11-13T18:13:48,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521688094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521688095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521688095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521688097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521688104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,256 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/02539b7ef98242cfb4b931c074c7306c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/02539b7ef98242cfb4b931c074c7306c 2024-11-13T18:13:48,265 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into 02539b7ef98242cfb4b931c074c7306c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:48,265 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:48,265 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521627774; duration=0sec 2024-11-13T18:13:48,265 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:48,265 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:13:48,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521688315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521688315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521688315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521688315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521688324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:13:48,459 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/1c319efe19b3453e9f4b08473b05997f 2024-11-13T18:13:48,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/ef31d3769b4e46169df4a9a33d201d5d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ef31d3769b4e46169df4a9a33d201d5d 2024-11-13T18:13:48,469 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ef31d3769b4e46169df4a9a33d201d5d, entries=150, sequenceid=81, filesize=30.2 K 2024-11-13T18:13:48,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/6382c2ebfd7343829b0dc4eafb138677 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6382c2ebfd7343829b0dc4eafb138677 2024-11-13T18:13:48,475 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6382c2ebfd7343829b0dc4eafb138677, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T18:13:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/1c319efe19b3453e9f4b08473b05997f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c319efe19b3453e9f4b08473b05997f 2024-11-13T18:13:48,481 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c319efe19b3453e9f4b08473b05997f, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T18:13:48,482 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fc0f52f0b3b547094f90472a203572fc in 707ms, sequenceid=81, compaction requested=false 2024-11-13T18:13:48,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:48,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:48,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-13T18:13:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-13T18:13:48,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-13T18:13:48,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1720 sec 2024-11-13T18:13:48,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.1770 sec 2024-11-13T18:13:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:48,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:13:48,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:48,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:48,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:48,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:48,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:48,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:48,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111393990f4b746541d0a534e3a760a17a3e_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521627967/Put/seqid=0 2024-11-13T18:13:48,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742227_1403 (size=14594) 2024-11-13T18:13:48,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:48,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111393990f4b746541d0a534e3a760a17a3e_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111393990f4b746541d0a534e3a760a17a3e_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:48,663 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/548ba38d1475431d9b657f9106c44e03, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:48,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/548ba38d1475431d9b657f9106c44e03 is 175, key is test_row_0/A:col10/1731521627967/Put/seqid=0 2024-11-13T18:13:48,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742228_1404 (size=39549) 2024-11-13T18:13:48,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521688659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521688671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521688674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521688675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521688681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521688784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521688784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521688794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521688794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521688794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521688994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521689004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521689004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521689015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521689015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,074 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/548ba38d1475431d9b657f9106c44e03 2024-11-13T18:13:49,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/ef0e92354c13470dadaf908ec68626a3 is 50, key is test_row_0/B:col10/1731521627967/Put/seqid=0 2024-11-13T18:13:49,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742229_1405 (size=12001) 2024-11-13T18:13:49,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/ef0e92354c13470dadaf908ec68626a3 2024-11-13T18:13:49,101 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:13:49,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/d8106c3fe0f645648042eccc93dc632b is 50, key is test_row_0/C:col10/1731521627967/Put/seqid=0 2024-11-13T18:13:49,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742230_1406 (size=12001) 2024-11-13T18:13:49,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/d8106c3fe0f645648042eccc93dc632b 2024-11-13T18:13:49,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/548ba38d1475431d9b657f9106c44e03 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/548ba38d1475431d9b657f9106c44e03 2024-11-13T18:13:49,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/548ba38d1475431d9b657f9106c44e03, entries=200, sequenceid=100, filesize=38.6 K 2024-11-13T18:13:49,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/ef0e92354c13470dadaf908ec68626a3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/ef0e92354c13470dadaf908ec68626a3 2024-11-13T18:13:49,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/ef0e92354c13470dadaf908ec68626a3, entries=150, sequenceid=100, filesize=11.7 K 2024-11-13T18:13:49,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/d8106c3fe0f645648042eccc93dc632b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d8106c3fe0f645648042eccc93dc632b 2024-11-13T18:13:49,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d8106c3fe0f645648042eccc93dc632b, entries=150, sequenceid=100, filesize=11.7 K 2024-11-13T18:13:49,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fc0f52f0b3b547094f90472a203572fc in 516ms, sequenceid=100, compaction requested=true 2024-11-13T18:13:49,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:49,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:49,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:49,156 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:49,156 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:49,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:49,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:49,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:49,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:49,157 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:49,157 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:13:49,157 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:49,157 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:49,157 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:13:49,157 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:49,157 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/02539b7ef98242cfb4b931c074c7306c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6382c2ebfd7343829b0dc4eafb138677, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/ef0e92354c13470dadaf908ec68626a3] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=35.3 K 2024-11-13T18:13:49,158 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/efd970fd26784d7d81e54900417b3878, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ef31d3769b4e46169df4a9a33d201d5d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/548ba38d1475431d9b657f9106c44e03] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=99.2 K 2024-11-13T18:13:49,158 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:49,158 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/efd970fd26784d7d81e54900417b3878, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ef31d3769b4e46169df4a9a33d201d5d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/548ba38d1475431d9b657f9106c44e03] 2024-11-13T18:13:49,158 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd970fd26784d7d81e54900417b3878, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731521627035 2024-11-13T18:13:49,158 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 02539b7ef98242cfb4b931c074c7306c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731521627035 2024-11-13T18:13:49,158 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6382c2ebfd7343829b0dc4eafb138677, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731521627214 2024-11-13T18:13:49,158 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef31d3769b4e46169df4a9a33d201d5d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731521627214 2024-11-13T18:13:49,159 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0e92354c13470dadaf908ec68626a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731521627967 2024-11-13T18:13:49,159 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 548ba38d1475431d9b657f9106c44e03, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731521627967 2024-11-13T18:13:49,171 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:49,171 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/a91efec4e8154658a135fe3c226f6912 is 50, key is test_row_0/B:col10/1731521627967/Put/seqid=0 2024-11-13T18:13:49,197 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:49,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742231_1407 (size=12207) 2024-11-13T18:13:49,217 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113aac5608186c04781bdc0041bb8e69d57_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:49,219 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113aac5608186c04781bdc0041bb8e69d57_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:49,220 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aac5608186c04781bdc0041bb8e69d57_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:49,235 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/a91efec4e8154658a135fe3c226f6912 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a91efec4e8154658a135fe3c226f6912 2024-11-13T18:13:49,248 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into a91efec4e8154658a135fe3c226f6912(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:49,248 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:49,248 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521629156; duration=0sec 2024-11-13T18:13:49,248 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:49,248 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:13:49,248 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:49,249 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:49,249 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/C is initiating minor compaction (all files) 2024-11-13T18:13:49,249 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/C in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:49,249 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a36f51b1961e435aa51118a4283afecd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c319efe19b3453e9f4b08473b05997f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d8106c3fe0f645648042eccc93dc632b] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=35.3 K 2024-11-13T18:13:49,250 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting a36f51b1961e435aa51118a4283afecd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731521627035 2024-11-13T18:13:49,252 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c319efe19b3453e9f4b08473b05997f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731521627214 2024-11-13T18:13:49,253 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d8106c3fe0f645648042eccc93dc632b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731521627967 2024-11-13T18:13:49,264 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:49,265 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/84510b9c4bf244eeb1606e60f642c073 is 50, key is test_row_0/C:col10/1731521627967/Put/seqid=0 2024-11-13T18:13:49,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742232_1408 (size=4469) 2024-11-13T18:13:49,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742233_1409 (size=12207) 2024-11-13T18:13:49,293 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/84510b9c4bf244eeb1606e60f642c073 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84510b9c4bf244eeb1606e60f642c073 2024-11-13T18:13:49,304 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/C of fc0f52f0b3b547094f90472a203572fc into 84510b9c4bf244eeb1606e60f642c073(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:49,304 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:49,304 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/C, priority=13, startTime=1731521629157; duration=0sec 2024-11-13T18:13:49,304 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:49,304 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:13:49,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:13:49,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:49,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:49,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:49,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:49,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:49,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:49,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:49,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f18bc751b7074e6cac4bbd4c5d3ea235_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521629319/Put/seqid=0 2024-11-13T18:13:49,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521689353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521689353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521689359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521689362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742234_1410 (size=14594) 2024-11-13T18:13:49,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:49,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521689369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,387 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f18bc751b7074e6cac4bbd4c5d3ea235_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f18bc751b7074e6cac4bbd4c5d3ea235_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:49,388 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/5d018941fb824601a5441f4d51f29c39, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:49,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/5d018941fb824601a5441f4d51f29c39 is 175, key is test_row_0/A:col10/1731521629319/Put/seqid=0 2024-11-13T18:13:49,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742235_1411 (size=39549) 2024-11-13T18:13:49,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/5d018941fb824601a5441f4d51f29c39 2024-11-13T18:13:49,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2010fc8e14de4c60977b5c05f9b1f0bf is 50, key is test_row_0/B:col10/1731521629319/Put/seqid=0 2024-11-13T18:13:49,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742236_1412 (size=12001) 2024-11-13T18:13:49,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2010fc8e14de4c60977b5c05f9b1f0bf 2024-11-13T18:13:49,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/f995ea63ec974c4ebfa964c9d11db64a is 50, key is test_row_0/C:col10/1731521629319/Put/seqid=0 2024-11-13T18:13:49,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742237_1413 (size=12001) 2024-11-13T18:13:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:13:49,455 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:49,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-13T18:13:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:13:49,457 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:49,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/f995ea63ec974c4ebfa964c9d11db64a 2024-11-13T18:13:49,461 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:49,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/5d018941fb824601a5441f4d51f29c39 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/5d018941fb824601a5441f4d51f29c39 2024-11-13T18:13:49,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/5d018941fb824601a5441f4d51f29c39, entries=200, sequenceid=119, filesize=38.6 K 2024-11-13T18:13:49,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2010fc8e14de4c60977b5c05f9b1f0bf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2010fc8e14de4c60977b5c05f9b1f0bf 2024-11-13T18:13:49,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2010fc8e14de4c60977b5c05f9b1f0bf, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T18:13:49,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/f995ea63ec974c4ebfa964c9d11db64a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f995ea63ec974c4ebfa964c9d11db64a 2024-11-13T18:13:49,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f995ea63ec974c4ebfa964c9d11db64a, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T18:13:49,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fc0f52f0b3b547094f90472a203572fc in 155ms, sequenceid=119, compaction requested=false 2024-11-13T18:13:49,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:49,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:13:49,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:49,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:49,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:49,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:49,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:49,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:49,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e8fd4417c2da4f1b94758bf6771a594b_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521629360/Put/seqid=0 2024-11-13T18:13:49,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742238_1414 (size=14794) 2024-11-13T18:13:49,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521689518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521689518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521689520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521689521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521689530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:13:49,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T18:13:49,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:49,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:49,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:49,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:49,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:49,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521689644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521689644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521689644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521689645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521689654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,696 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#349 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:49,697 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b969f66f170a4109b2cd09829fe7862b is 175, key is test_row_0/A:col10/1731521627967/Put/seqid=0 2024-11-13T18:13:49,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742239_1415 (size=31161) 2024-11-13T18:13:49,735 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b969f66f170a4109b2cd09829fe7862b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b969f66f170a4109b2cd09829fe7862b 2024-11-13T18:13:49,743 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into b969f66f170a4109b2cd09829fe7862b(size=30.4 K), total size for store is 69.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:49,743 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:49,743 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521629156; duration=0sec 2024-11-13T18:13:49,743 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:49,743 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:13:49,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T18:13:49,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:49,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:49,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:49,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:49,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:13:49,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521689865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521689865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521689865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521689865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521689874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:49,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:49,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:49,906 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:49,910 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e8fd4417c2da4f1b94758bf6771a594b_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e8fd4417c2da4f1b94758bf6771a594b_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:49,911 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/dd7e79910e024817a6de7288702279c4, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:49,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/dd7e79910e024817a6de7288702279c4 is 175, key is test_row_0/A:col10/1731521629360/Put/seqid=0 2024-11-13T18:13:49,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T18:13:49,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:49,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:49,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:49,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:49,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742240_1416 (size=39749) 2024-11-13T18:13:50,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T18:13:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:13:50,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521690184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521690185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521690185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521690185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521690194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T18:13:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:50,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:50,347 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/dd7e79910e024817a6de7288702279c4 2024-11-13T18:13:50,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/8a127a4c2c5044dc95f49ca74fee57c5 is 50, key is test_row_0/B:col10/1731521629360/Put/seqid=0 2024-11-13T18:13:50,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742241_1417 (size=12151) 2024-11-13T18:13:50,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/8a127a4c2c5044dc95f49ca74fee57c5 2024-11-13T18:13:50,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/dac98bf7088d48edb3c77c00537eb054 is 50, key is test_row_0/C:col10/1731521629360/Put/seqid=0 2024-11-13T18:13:50,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742242_1418 (size=12151) 2024-11-13T18:13:50,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/dac98bf7088d48edb3c77c00537eb054 2024-11-13T18:13:50,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T18:13:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:50,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:50,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:50,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/dd7e79910e024817a6de7288702279c4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/dd7e79910e024817a6de7288702279c4 2024-11-13T18:13:50,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/dd7e79910e024817a6de7288702279c4, entries=200, sequenceid=135, filesize=38.8 K 2024-11-13T18:13:50,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/8a127a4c2c5044dc95f49ca74fee57c5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/8a127a4c2c5044dc95f49ca74fee57c5 2024-11-13T18:13:50,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/8a127a4c2c5044dc95f49ca74fee57c5, entries=150, sequenceid=135, filesize=11.9 K 2024-11-13T18:13:50,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/dac98bf7088d48edb3c77c00537eb054 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/dac98bf7088d48edb3c77c00537eb054 2024-11-13T18:13:50,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/dac98bf7088d48edb3c77c00537eb054, entries=150, sequenceid=135, filesize=11.9 K 2024-11-13T18:13:50,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fc0f52f0b3b547094f90472a203572fc in 929ms, sequenceid=135, compaction requested=true 2024-11-13T18:13:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:50,414 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:50,414 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:50,415 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:50,415 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:13:50,415 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,415 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a91efec4e8154658a135fe3c226f6912, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2010fc8e14de4c60977b5c05f9b1f0bf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/8a127a4c2c5044dc95f49ca74fee57c5] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=35.5 K 2024-11-13T18:13:50,415 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:50,415 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:13:50,415 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,416 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b969f66f170a4109b2cd09829fe7862b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/5d018941fb824601a5441f4d51f29c39, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/dd7e79910e024817a6de7288702279c4] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=107.9 K 2024-11-13T18:13:50,416 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,416 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b969f66f170a4109b2cd09829fe7862b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/5d018941fb824601a5441f4d51f29c39, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/dd7e79910e024817a6de7288702279c4] 2024-11-13T18:13:50,416 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting a91efec4e8154658a135fe3c226f6912, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731521627967 2024-11-13T18:13:50,416 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b969f66f170a4109b2cd09829fe7862b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731521627967 2024-11-13T18:13:50,416 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 2010fc8e14de4c60977b5c05f9b1f0bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731521628655 2024-11-13T18:13:50,416 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d018941fb824601a5441f4d51f29c39, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731521628655 2024-11-13T18:13:50,417 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a127a4c2c5044dc95f49ca74fee57c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731521629350 2024-11-13T18:13:50,417 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd7e79910e024817a6de7288702279c4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731521629350 2024-11-13T18:13:50,435 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:50,435 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/d2d50797e64849e0a0d085b075305bc3 is 50, key is test_row_0/B:col10/1731521629360/Put/seqid=0 2024-11-13T18:13:50,437 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:50,451 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113566ac86f43fb411d9134ff916c2e305d_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:50,453 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113566ac86f43fb411d9134ff916c2e305d_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:50,454 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113566ac86f43fb411d9134ff916c2e305d_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:50,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742243_1419 (size=12459) 2024-11-13T18:13:50,479 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/d2d50797e64849e0a0d085b075305bc3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/d2d50797e64849e0a0d085b075305bc3 2024-11-13T18:13:50,484 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into d2d50797e64849e0a0d085b075305bc3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:50,484 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:50,484 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521630414; duration=0sec 2024-11-13T18:13:50,484 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:50,484 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:13:50,484 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:50,486 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:50,486 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/C is initiating minor compaction (all files) 2024-11-13T18:13:50,486 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/C in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,486 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84510b9c4bf244eeb1606e60f642c073, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f995ea63ec974c4ebfa964c9d11db64a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/dac98bf7088d48edb3c77c00537eb054] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=35.5 K 2024-11-13T18:13:50,488 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 84510b9c4bf244eeb1606e60f642c073, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731521627967 2024-11-13T18:13:50,489 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f995ea63ec974c4ebfa964c9d11db64a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731521628655 2024-11-13T18:13:50,489 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting dac98bf7088d48edb3c77c00537eb054, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731521629350 2024-11-13T18:13:50,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742244_1420 (size=4469) 2024-11-13T18:13:50,503 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#C#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:50,504 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/df27bd3d629c4517830cee5099812a18 is 50, key is test_row_0/C:col10/1731521629360/Put/seqid=0 2024-11-13T18:13:50,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742245_1421 (size=12459) 2024-11-13T18:13:50,522 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/df27bd3d629c4517830cee5099812a18 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/df27bd3d629c4517830cee5099812a18 2024-11-13T18:13:50,528 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/C of fc0f52f0b3b547094f90472a203572fc into df27bd3d629c4517830cee5099812a18(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:50,528 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:50,528 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/C, priority=13, startTime=1731521630414; duration=0sec 2024-11-13T18:13:50,528 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:50,528 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:13:50,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T18:13:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:50,547 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:13:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:50,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113502a62194d804de29a401eb5c00325c7_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521629527/Put/seqid=0 2024-11-13T18:13:50,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742246_1422 (size=12304) 2024-11-13T18:13:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:13:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:50,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:50,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521690748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521690749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521690751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521690752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521690753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521690874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521690874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521690876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521690876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521690877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:50,894 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#358 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:50,894 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/c027b530a3af4513ba304ce9a2d78df8 is 175, key is test_row_0/A:col10/1731521629360/Put/seqid=0 2024-11-13T18:13:50,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742247_1423 (size=31413) 2024-11-13T18:13:50,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:50,982 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113502a62194d804de29a401eb5c00325c7_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113502a62194d804de29a401eb5c00325c7_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:50,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b95c72a0082543f389f1abaf43d12e11, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:50,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b95c72a0082543f389f1abaf43d12e11 is 175, key is test_row_0/A:col10/1731521629527/Put/seqid=0 2024-11-13T18:13:50,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742248_1424 (size=31105) 2024-11-13T18:13:51,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521691094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521691094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521691094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521691094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521691094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,311 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/c027b530a3af4513ba304ce9a2d78df8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c027b530a3af4513ba304ce9a2d78df8 2024-11-13T18:13:51,316 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into c027b530a3af4513ba304ce9a2d78df8(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:51,316 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:51,316 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521630414; duration=0sec 2024-11-13T18:13:51,316 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:51,316 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:13:51,390 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b95c72a0082543f389f1abaf43d12e11 2024-11-13T18:13:51,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/c48995a7746744f58841247a7c76c139 is 50, key is test_row_0/B:col10/1731521629527/Put/seqid=0 2024-11-13T18:13:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521691414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521691415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521691416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521691417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521691417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742249_1425 (size=12151) 2024-11-13T18:13:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:13:51,837 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/c48995a7746744f58841247a7c76c139 2024-11-13T18:13:51,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/84e002eec6454e2e9b8b5dc20946a383 is 50, key is test_row_0/C:col10/1731521629527/Put/seqid=0 2024-11-13T18:13:51,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742250_1426 (size=12151) 2024-11-13T18:13:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521691924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521691924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521691934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521691934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:51,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521691934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:52,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/84e002eec6454e2e9b8b5dc20946a383 2024-11-13T18:13:52,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b95c72a0082543f389f1abaf43d12e11 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b95c72a0082543f389f1abaf43d12e11 2024-11-13T18:13:52,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b95c72a0082543f389f1abaf43d12e11, entries=150, sequenceid=154, filesize=30.4 K 2024-11-13T18:13:52,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/c48995a7746744f58841247a7c76c139 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c48995a7746744f58841247a7c76c139 2024-11-13T18:13:52,293 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c48995a7746744f58841247a7c76c139, entries=150, sequenceid=154, filesize=11.9 K 2024-11-13T18:13:52,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/84e002eec6454e2e9b8b5dc20946a383 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84e002eec6454e2e9b8b5dc20946a383 2024-11-13T18:13:52,300 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84e002eec6454e2e9b8b5dc20946a383, entries=150, sequenceid=154, filesize=11.9 K 2024-11-13T18:13:52,301 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for fc0f52f0b3b547094f90472a203572fc in 1754ms, sequenceid=154, compaction requested=false 2024-11-13T18:13:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-13T18:13:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-13T18:13:52,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-13T18:13:52,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8410 sec 2024-11-13T18:13:52,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.8500 sec 2024-11-13T18:13:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:52,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:13:52,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:52,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:52,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:52,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:52,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:52,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:52,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132076668e6ead44aea40c8cbcd6f96d8a_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521630746/Put/seqid=0 2024-11-13T18:13:52,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742251_1427 (size=14794) 2024-11-13T18:13:52,965 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:52,970 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132076668e6ead44aea40c8cbcd6f96d8a_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132076668e6ead44aea40c8cbcd6f96d8a_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:52,971 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/c6c6dec3385c4cadb01aed2fe0aca08a, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:52,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/c6c6dec3385c4cadb01aed2fe0aca08a is 175, key is test_row_0/A:col10/1731521630746/Put/seqid=0 2024-11-13T18:13:52,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742252_1428 (size=39749) 2024-11-13T18:13:52,982 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/c6c6dec3385c4cadb01aed2fe0aca08a 2024-11-13T18:13:52,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521692968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:52,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521692969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521692970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521692979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:52,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/19c62ced4df44fd1abf02034e2a19acf is 50, key is test_row_0/B:col10/1731521630746/Put/seqid=0 2024-11-13T18:13:52,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521692981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742253_1429 (size=12151) 2024-11-13T18:13:53,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/19c62ced4df44fd1abf02034e2a19acf 2024-11-13T18:13:53,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/62e8d8bd7c0842078d9e48426158a58a is 50, key is test_row_0/C:col10/1731521630746/Put/seqid=0 2024-11-13T18:13:53,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742254_1430 (size=12151) 2024-11-13T18:13:53,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/62e8d8bd7c0842078d9e48426158a58a 2024-11-13T18:13:53,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/c6c6dec3385c4cadb01aed2fe0aca08a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c6c6dec3385c4cadb01aed2fe0aca08a 2024-11-13T18:13:53,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c6c6dec3385c4cadb01aed2fe0aca08a, entries=200, sequenceid=173, filesize=38.8 K 2024-11-13T18:13:53,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/19c62ced4df44fd1abf02034e2a19acf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/19c62ced4df44fd1abf02034e2a19acf 2024-11-13T18:13:53,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/19c62ced4df44fd1abf02034e2a19acf, entries=150, sequenceid=173, filesize=11.9 K 2024-11-13T18:13:53,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/62e8d8bd7c0842078d9e48426158a58a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/62e8d8bd7c0842078d9e48426158a58a 2024-11-13T18:13:53,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/62e8d8bd7c0842078d9e48426158a58a, entries=150, sequenceid=173, filesize=11.9 K 2024-11-13T18:13:53,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fc0f52f0b3b547094f90472a203572fc in 127ms, sequenceid=173, compaction requested=true 2024-11-13T18:13:53,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:53,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:53,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:53,066 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:53,067 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:53,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:53,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:53,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:53,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:53,068 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:53,068 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:53,068 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:13:53,068 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:13:53,068 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:53,068 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:53,068 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/d2d50797e64849e0a0d085b075305bc3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c48995a7746744f58841247a7c76c139, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/19c62ced4df44fd1abf02034e2a19acf] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=35.9 K 2024-11-13T18:13:53,068 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c027b530a3af4513ba304ce9a2d78df8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b95c72a0082543f389f1abaf43d12e11, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c6c6dec3385c4cadb01aed2fe0aca08a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=99.9 K 2024-11-13T18:13:53,068 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:53,068 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c027b530a3af4513ba304ce9a2d78df8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b95c72a0082543f389f1abaf43d12e11, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c6c6dec3385c4cadb01aed2fe0aca08a] 2024-11-13T18:13:53,068 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d2d50797e64849e0a0d085b075305bc3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731521629350 2024-11-13T18:13:53,069 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c48995a7746744f58841247a7c76c139, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731521629517 2024-11-13T18:13:53,069 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c027b530a3af4513ba304ce9a2d78df8, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731521629350 2024-11-13T18:13:53,069 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c62ced4df44fd1abf02034e2a19acf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731521630746 2024-11-13T18:13:53,071 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b95c72a0082543f389f1abaf43d12e11, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731521629517 2024-11-13T18:13:53,071 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6c6dec3385c4cadb01aed2fe0aca08a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731521630746 2024-11-13T18:13:53,079 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:53,079 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/497366dec20a4d2f96620cb833f1ff11 is 50, key is test_row_0/B:col10/1731521630746/Put/seqid=0 2024-11-13T18:13:53,089 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:53,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742255_1431 (size=12561) 2024-11-13T18:13:53,091 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113586f19b42a124c2280395fefc3327da2_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:53,094 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113586f19b42a124c2280395fefc3327da2_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:53,094 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113586f19b42a124c2280395fefc3327da2_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:53,097 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/497366dec20a4d2f96620cb833f1ff11 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/497366dec20a4d2f96620cb833f1ff11 2024-11-13T18:13:53,102 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into 497366dec20a4d2f96620cb833f1ff11(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:53,102 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:53,102 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521633066; duration=0sec 2024-11-13T18:13:53,103 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:53,103 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:13:53,103 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:53,109 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:53,109 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/C is initiating minor compaction (all files) 2024-11-13T18:13:53,109 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/C in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:53,109 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/df27bd3d629c4517830cee5099812a18, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84e002eec6454e2e9b8b5dc20946a383, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/62e8d8bd7c0842078d9e48426158a58a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=35.9 K 2024-11-13T18:13:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:53,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:13:53,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:53,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:53,111 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting df27bd3d629c4517830cee5099812a18, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731521629350 2024-11-13T18:13:53,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:53,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:53,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:53,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:53,112 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 84e002eec6454e2e9b8b5dc20946a383, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731521629517 2024-11-13T18:13:53,112 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e8d8bd7c0842078d9e48426158a58a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731521630746 2024-11-13T18:13:53,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742256_1432 (size=4469) 2024-11-13T18:13:53,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136de96acab8eb4f1387b74c33c30f2482_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521632978/Put/seqid=0 2024-11-13T18:13:53,133 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#C#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:53,133 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/74deeb3966af46d69f3cac808c446774 is 50, key is test_row_0/C:col10/1731521630746/Put/seqid=0 2024-11-13T18:13:53,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742257_1433 (size=12304) 2024-11-13T18:13:53,147 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:53,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,154 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136de96acab8eb4f1387b74c33c30f2482_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136de96acab8eb4f1387b74c33c30f2482_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521693147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742258_1434 (size=12561) 2024-11-13T18:13:53,156 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:53,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0 is 175, key is test_row_0/A:col10/1731521632978/Put/seqid=0 2024-11-13T18:13:53,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521693149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,162 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/74deeb3966af46d69f3cac808c446774 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74deeb3966af46d69f3cac808c446774 2024-11-13T18:13:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742259_1435 (size=31105) 2024-11-13T18:13:53,166 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0 2024-11-13T18:13:53,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521693151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521693152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,170 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/C of fc0f52f0b3b547094f90472a203572fc into 74deeb3966af46d69f3cac808c446774(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:53,170 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:53,170 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/C, priority=13, startTime=1731521633067; duration=0sec 2024-11-13T18:13:53,170 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:53,170 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:13:53,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521693154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/6ddd4dd4adad4566894fb3e5b0870777 is 50, key is test_row_0/B:col10/1731521632978/Put/seqid=0 2024-11-13T18:13:53,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742260_1436 (size=12151) 2024-11-13T18:13:53,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/6ddd4dd4adad4566894fb3e5b0870777 2024-11-13T18:13:53,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/32a50d325629475e967f531e9cebfcbf is 50, key is test_row_0/C:col10/1731521632978/Put/seqid=0 2024-11-13T18:13:53,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742261_1437 (size=12151) 2024-11-13T18:13:53,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/32a50d325629475e967f531e9cebfcbf 2024-11-13T18:13:53,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0 2024-11-13T18:13:53,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0, entries=150, sequenceid=190, filesize=30.4 K 2024-11-13T18:13:53,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/6ddd4dd4adad4566894fb3e5b0870777 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6ddd4dd4adad4566894fb3e5b0870777 2024-11-13T18:13:53,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6ddd4dd4adad4566894fb3e5b0870777, entries=150, sequenceid=190, filesize=11.9 K 2024-11-13T18:13:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/32a50d325629475e967f531e9cebfcbf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/32a50d325629475e967f531e9cebfcbf 2024-11-13T18:13:53,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/32a50d325629475e967f531e9cebfcbf, entries=150, sequenceid=190, filesize=11.9 K 2024-11-13T18:13:53,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fc0f52f0b3b547094f90472a203572fc in 122ms, sequenceid=190, compaction requested=false 2024-11-13T18:13:53,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:53,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:53,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:13:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:53,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133a2c66deb4774cf396b6a5207f4160b5_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521633135/Put/seqid=0 2024-11-13T18:13:53,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742262_1438 (size=14794) 2024-11-13T18:13:53,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521693297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521693299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521693300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521693303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521693305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521693424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521693424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521693425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521693425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521693425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,529 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#367 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:53,530 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/7b0bfe58499240ad92e63f4f1dfa45a2 is 175, key is test_row_0/A:col10/1731521630746/Put/seqid=0 2024-11-13T18:13:53,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742263_1439 (size=31515) 2024-11-13T18:13:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:13:53,615 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:53,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-13T18:13:53,618 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:53,619 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:53,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:53,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:13:53,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521693645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521693646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521693646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521693646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521693646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,689 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:53,696 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133a2c66deb4774cf396b6a5207f4160b5_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133a2c66deb4774cf396b6a5207f4160b5_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:53,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b262be014e9b4738b254912f4482a7d4, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:53,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b262be014e9b4738b254912f4482a7d4 is 175, key is test_row_0/A:col10/1731521633135/Put/seqid=0 2024-11-13T18:13:53,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742264_1440 (size=39749) 2024-11-13T18:13:53,710 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b262be014e9b4738b254912f4482a7d4 2024-11-13T18:13:53,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/4f909f109c104e3796b178ef12b8cfb7 is 50, key is test_row_0/B:col10/1731521633135/Put/seqid=0 2024-11-13T18:13:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:13:53,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742265_1441 (size=12151) 2024-11-13T18:13:53,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:13:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:53,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:53,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:13:53,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:53,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:53,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:53,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:53,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:13:53,948 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/7b0bfe58499240ad92e63f4f1dfa45a2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/7b0bfe58499240ad92e63f4f1dfa45a2 2024-11-13T18:13:53,962 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into 7b0bfe58499240ad92e63f4f1dfa45a2(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:53,962 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:53,963 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521633066; duration=0sec 2024-11-13T18:13:53,963 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:53,963 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:13:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521693964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521693965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521693965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521693965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521693965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:13:54,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:54,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:54,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:54,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:54,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:54,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/4f909f109c104e3796b178ef12b8cfb7 2024-11-13T18:13:54,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/1d0dd7c1784047128ffd86d275dcb087 is 50, key is test_row_0/C:col10/1731521633135/Put/seqid=0 2024-11-13T18:13:54,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742266_1442 (size=12151) 2024-11-13T18:13:54,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/1d0dd7c1784047128ffd86d275dcb087 2024-11-13T18:13:54,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b262be014e9b4738b254912f4482a7d4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b262be014e9b4738b254912f4482a7d4 2024-11-13T18:13:54,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b262be014e9b4738b254912f4482a7d4, entries=200, sequenceid=208, filesize=38.8 K 2024-11-13T18:13:54,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/4f909f109c104e3796b178ef12b8cfb7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/4f909f109c104e3796b178ef12b8cfb7 2024-11-13T18:13:54,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/4f909f109c104e3796b178ef12b8cfb7, entries=150, sequenceid=208, filesize=11.9 K 2024-11-13T18:13:54,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/1d0dd7c1784047128ffd86d275dcb087 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1d0dd7c1784047128ffd86d275dcb087 2024-11-13T18:13:54,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1d0dd7c1784047128ffd86d275dcb087, entries=150, sequenceid=208, filesize=11.9 K 2024-11-13T18:13:54,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fc0f52f0b3b547094f90472a203572fc in 913ms, sequenceid=208, compaction requested=true 2024-11-13T18:13:54,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:54,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:54,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:54,185 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:54,186 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:54,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:54,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:54,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:54,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:54,186 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:54,186 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:54,186 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:13:54,186 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:13:54,187 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:54,187 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:54,187 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/497366dec20a4d2f96620cb833f1ff11, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6ddd4dd4adad4566894fb3e5b0870777, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/4f909f109c104e3796b178ef12b8cfb7] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=36.0 K 2024-11-13T18:13:54,187 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/7b0bfe58499240ad92e63f4f1dfa45a2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b262be014e9b4738b254912f4482a7d4] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=100.0 K 2024-11-13T18:13:54,187 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:54,187 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/7b0bfe58499240ad92e63f4f1dfa45a2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b262be014e9b4738b254912f4482a7d4] 2024-11-13T18:13:54,188 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 497366dec20a4d2f96620cb833f1ff11, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731521630746 2024-11-13T18:13:54,188 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0bfe58499240ad92e63f4f1dfa45a2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731521630746 2024-11-13T18:13:54,188 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ddd4dd4adad4566894fb3e5b0870777, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731521632978 2024-11-13T18:13:54,188 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bd0ef00bd814c3ba9a6bc360ae7a4e0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731521632978 2024-11-13T18:13:54,189 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f909f109c104e3796b178ef12b8cfb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731521633135 2024-11-13T18:13:54,189 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b262be014e9b4738b254912f4482a7d4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731521633135 2024-11-13T18:13:54,195 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:54,198 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:54,199 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/87e1a5953eb04ddc9ca14a2c8e8b8721 is 50, key is test_row_0/B:col10/1731521633135/Put/seqid=0 2024-11-13T18:13:54,201 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113393c1e7d64014b80bf8280908344172b_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:54,203 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113393c1e7d64014b80bf8280908344172b_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:54,204 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113393c1e7d64014b80bf8280908344172b_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:54,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742268_1444 (size=4469) 2024-11-13T18:13:54,215 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#375 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:54,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742267_1443 (size=12663) 2024-11-13T18:13:54,215 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/cfd734ce8edb4c06a92656567cf087ed is 175, key is test_row_0/A:col10/1731521633135/Put/seqid=0 2024-11-13T18:13:54,221 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/87e1a5953eb04ddc9ca14a2c8e8b8721 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/87e1a5953eb04ddc9ca14a2c8e8b8721 2024-11-13T18:13:54,236 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into 87e1a5953eb04ddc9ca14a2c8e8b8721(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:54,236 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:54,236 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521634185; duration=0sec 2024-11-13T18:13:54,236 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:54,236 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:13:54,236 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:54,237 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:54,237 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/C is initiating minor compaction (all files) 2024-11-13T18:13:54,237 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/C in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:54,237 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74deeb3966af46d69f3cac808c446774, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/32a50d325629475e967f531e9cebfcbf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1d0dd7c1784047128ffd86d275dcb087] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=36.0 K 2024-11-13T18:13:54,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:13:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:54,237 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:13:54,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:54,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:54,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:54,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:54,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:54,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:54,238 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 74deeb3966af46d69f3cac808c446774, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731521630746 2024-11-13T18:13:54,238 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 32a50d325629475e967f531e9cebfcbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731521632978 2024-11-13T18:13:54,239 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d0dd7c1784047128ffd86d275dcb087, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731521633135 2024-11-13T18:13:54,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742269_1445 (size=31617) 2024-11-13T18:13:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:13:54,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113da0895753285491199379e43a28e901b_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521633283/Put/seqid=0 2024-11-13T18:13:54,257 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#C#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:54,257 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/07bc165bab6146dd9f7abe3f913e8403 is 50, key is test_row_0/C:col10/1731521633135/Put/seqid=0 2024-11-13T18:13:54,263 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/cfd734ce8edb4c06a92656567cf087ed as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/cfd734ce8edb4c06a92656567cf087ed 2024-11-13T18:13:54,270 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into cfd734ce8edb4c06a92656567cf087ed(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:54,270 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:54,270 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521634185; duration=0sec 2024-11-13T18:13:54,270 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:54,270 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:13:54,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742270_1446 (size=12304) 2024-11-13T18:13:54,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:54,312 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113da0895753285491199379e43a28e901b_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113da0895753285491199379e43a28e901b_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:54,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:54,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e is 175, key is test_row_0/A:col10/1731521633283/Put/seqid=0 2024-11-13T18:13:54,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742271_1447 (size=12663) 2024-11-13T18:13:54,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742272_1448 (size=31105) 2024-11-13T18:13:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:54,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521694509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521694551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521694551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521694551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521694552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521694664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521694674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521694674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521694674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521694674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,723 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/07bc165bab6146dd9f7abe3f913e8403 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/07bc165bab6146dd9f7abe3f913e8403 2024-11-13T18:13:54,728 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/C of fc0f52f0b3b547094f90472a203572fc into 07bc165bab6146dd9f7abe3f913e8403(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:54,728 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:54,728 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/C, priority=13, startTime=1731521634186; duration=0sec 2024-11-13T18:13:54,728 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:54,728 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:13:54,741 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e 2024-11-13T18:13:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/b85edce65eff4786b61c123b0330469e is 50, key is test_row_0/B:col10/1731521633283/Put/seqid=0 2024-11-13T18:13:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:13:54,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742273_1449 (size=12151) 2024-11-13T18:13:54,755 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/b85edce65eff4786b61c123b0330469e 2024-11-13T18:13:54,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/d49bfd40718b4281be92ef57c53702b5 is 50, key is test_row_0/C:col10/1731521633283/Put/seqid=0 2024-11-13T18:13:54,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742274_1450 (size=12151) 2024-11-13T18:13:54,772 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/d49bfd40718b4281be92ef57c53702b5 2024-11-13T18:13:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e 2024-11-13T18:13:54,783 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e, entries=150, sequenceid=225, filesize=30.4 K 2024-11-13T18:13:54,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/b85edce65eff4786b61c123b0330469e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b85edce65eff4786b61c123b0330469e 2024-11-13T18:13:54,798 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b85edce65eff4786b61c123b0330469e, entries=150, sequenceid=225, filesize=11.9 K 2024-11-13T18:13:54,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/d49bfd40718b4281be92ef57c53702b5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d49bfd40718b4281be92ef57c53702b5 2024-11-13T18:13:54,806 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d49bfd40718b4281be92ef57c53702b5, entries=150, sequenceid=225, filesize=11.9 K 2024-11-13T18:13:54,807 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fc0f52f0b3b547094f90472a203572fc in 570ms, sequenceid=225, compaction requested=false 2024-11-13T18:13:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-13T18:13:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-13T18:13:54,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-13T18:13:54,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1980 sec 2024-11-13T18:13:54,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.2040 sec 2024-11-13T18:13:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:54,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:13:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:54,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113af931412539844179107086658be06dc_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521634524/Put/seqid=0 2024-11-13T18:13:54,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742275_1451 (size=14794) 2024-11-13T18:13:54,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521694918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521694918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521694929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521694930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521694930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521695034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521695044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521695054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521695054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521695054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521695254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521695264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521695275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521695275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521695275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,299 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:55,303 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113af931412539844179107086658be06dc_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113af931412539844179107086658be06dc_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:55,304 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/67ea364880a94d8ab3d63212c45ab65b, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:55,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/67ea364880a94d8ab3d63212c45ab65b is 175, key is test_row_0/A:col10/1731521634524/Put/seqid=0 2024-11-13T18:13:55,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742276_1452 (size=39749) 2024-11-13T18:13:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521695564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521695584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521695584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521695584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521695584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:55,732 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/67ea364880a94d8ab3d63212c45ab65b 2024-11-13T18:13:55,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2d71e6fe8ab444dc84082798549cf5f3 is 50, key is test_row_0/B:col10/1731521634524/Put/seqid=0 2024-11-13T18:13:55,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742277_1453 (size=12151) 2024-11-13T18:13:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:13:55,776 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:55,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-13T18:13:55,781 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:55,782 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:55,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:13:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:13:55,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T18:13:55,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:55,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:55,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:55,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:55,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:56,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T18:13:56,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:56,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:56,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:56,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:56,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:56,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521696084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:13:56,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521696094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:56,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521696094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:56,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521696094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:56,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521696094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:56,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2d71e6fe8ab444dc84082798549cf5f3 2024-11-13T18:13:56,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/390f851ab7cd450f9c9ca724374057aa is 50, key is test_row_0/C:col10/1731521634524/Put/seqid=0 2024-11-13T18:13:56,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742278_1454 (size=12151) 2024-11-13T18:13:56,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/390f851ab7cd450f9c9ca724374057aa 2024-11-13T18:13:56,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/67ea364880a94d8ab3d63212c45ab65b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/67ea364880a94d8ab3d63212c45ab65b 2024-11-13T18:13:56,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/67ea364880a94d8ab3d63212c45ab65b, entries=200, sequenceid=245, filesize=38.8 K 2024-11-13T18:13:56,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2d71e6fe8ab444dc84082798549cf5f3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2d71e6fe8ab444dc84082798549cf5f3 2024-11-13T18:13:56,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2d71e6fe8ab444dc84082798549cf5f3, entries=150, sequenceid=245, filesize=11.9 K 2024-11-13T18:13:56,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/390f851ab7cd450f9c9ca724374057aa as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/390f851ab7cd450f9c9ca724374057aa 2024-11-13T18:13:56,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/390f851ab7cd450f9c9ca724374057aa, entries=150, sequenceid=245, filesize=11.9 K 2024-11-13T18:13:56,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for fc0f52f0b3b547094f90472a203572fc in 1343ms, sequenceid=245, compaction requested=true 2024-11-13T18:13:56,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:56,221 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:56,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:56,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:56,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:56,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:56,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:56,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:13:56,222 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:56,222 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:13:56,222 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:56,223 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/cfd734ce8edb4c06a92656567cf087ed, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/67ea364880a94d8ab3d63212c45ab65b] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=100.1 K 2024-11-13T18:13:56,223 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:56,223 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/cfd734ce8edb4c06a92656567cf087ed, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/67ea364880a94d8ab3d63212c45ab65b] 2024-11-13T18:13:56,223 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting cfd734ce8edb4c06a92656567cf087ed, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731521633135 2024-11-13T18:13:56,223 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting eb70b1ec1b9d4d83b1a46b7b3bc4c27e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731521633283 2024-11-13T18:13:56,224 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 67ea364880a94d8ab3d63212c45ab65b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731521634506 2024-11-13T18:13:56,224 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:56,231 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:56,232 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:13:56,232 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:56,232 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/87e1a5953eb04ddc9ca14a2c8e8b8721, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b85edce65eff4786b61c123b0330469e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2d71e6fe8ab444dc84082798549cf5f3] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=36.1 K 2024-11-13T18:13:56,233 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87e1a5953eb04ddc9ca14a2c8e8b8721, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731521633135 2024-11-13T18:13:56,233 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b85edce65eff4786b61c123b0330469e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731521633283 2024-11-13T18:13:56,235 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d71e6fe8ab444dc84082798549cf5f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731521634524 2024-11-13T18:13:56,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T18:13:56,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:56,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:13:56,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:56,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:56,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:56,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:56,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:56,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:56,257 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:56,271 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:56,273 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/233ad71fe17c4c25a52ba07d9a9ed476 is 50, key is test_row_0/B:col10/1731521634524/Put/seqid=0 2024-11-13T18:13:56,277 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111366f1ab2bae694c7f8224d493a42aa9f4_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:56,279 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111366f1ab2bae694c7f8224d493a42aa9f4_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:56,279 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111366f1ab2bae694c7f8224d493a42aa9f4_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:56,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134a6b1864d3a34afdb22f299d7ad01563_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521634927/Put/seqid=0 2024-11-13T18:13:56,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742279_1455 (size=12765) 2024-11-13T18:13:56,325 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/233ad71fe17c4c25a52ba07d9a9ed476 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/233ad71fe17c4c25a52ba07d9a9ed476 2024-11-13T18:13:56,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742280_1456 (size=4469) 2024-11-13T18:13:56,335 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into 233ad71fe17c4c25a52ba07d9a9ed476(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:56,335 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:56,336 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521636222; duration=0sec 2024-11-13T18:13:56,336 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:56,336 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#384 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:56,336 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:13:56,336 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:56,337 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/658a01b5099a4a6d84e0acdab8092205 is 175, key is test_row_0/A:col10/1731521634524/Put/seqid=0 2024-11-13T18:13:56,338 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:56,338 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/C is initiating minor compaction (all files) 2024-11-13T18:13:56,338 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/C in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:56,338 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/07bc165bab6146dd9f7abe3f913e8403, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d49bfd40718b4281be92ef57c53702b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/390f851ab7cd450f9c9ca724374057aa] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=36.1 K 2024-11-13T18:13:56,339 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07bc165bab6146dd9f7abe3f913e8403, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731521633135 2024-11-13T18:13:56,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742281_1457 (size=12454) 2024-11-13T18:13:56,340 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d49bfd40718b4281be92ef57c53702b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731521633283 2024-11-13T18:13:56,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,345 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134a6b1864d3a34afdb22f299d7ad01563_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134a6b1864d3a34afdb22f299d7ad01563_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:56,345 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 390f851ab7cd450f9c9ca724374057aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731521634524 2024-11-13T18:13:56,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/81ef485750064d68bce4e5b93dbc2a5d, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:56,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/81ef485750064d68bce4e5b93dbc2a5d is 175, key is test_row_0/A:col10/1731521634927/Put/seqid=0 2024-11-13T18:13:56,359 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#C#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:56,363 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/e9c21d205bac459ba938e0c4a843bbbf is 50, key is test_row_0/C:col10/1731521634524/Put/seqid=0 2024-11-13T18:13:56,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742283_1459 (size=31255) 2024-11-13T18:13:56,370 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/81ef485750064d68bce4e5b93dbc2a5d 2024-11-13T18:13:56,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742282_1458 (size=31719) 2024-11-13T18:13:56,400 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/658a01b5099a4a6d84e0acdab8092205 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/658a01b5099a4a6d84e0acdab8092205 2024-11-13T18:13:56,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/486ceced31e7447393637fb3d223d659 is 50, key is test_row_0/B:col10/1731521634927/Put/seqid=0 2024-11-13T18:13:56,407 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into 658a01b5099a4a6d84e0acdab8092205(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:56,407 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:56,407 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521636221; duration=0sec 2024-11-13T18:13:56,407 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:56,407 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:13:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:13:56,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742284_1460 (size=12765) 2024-11-13T18:13:56,434 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/e9c21d205bac459ba938e0c4a843bbbf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e9c21d205bac459ba938e0c4a843bbbf 2024-11-13T18:13:56,441 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/C of fc0f52f0b3b547094f90472a203572fc into e9c21d205bac459ba938e0c4a843bbbf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:56,441 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:56,441 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/C, priority=13, startTime=1731521636222; duration=0sec 2024-11-13T18:13:56,442 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:56,442 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:13:56,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742285_1461 (size=12301) 2024-11-13T18:13:56,450 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/486ceced31e7447393637fb3d223d659 2024-11-13T18:13:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/7656ba08010849a9a8c8779dc08d202e is 50, key is test_row_0/C:col10/1731521634927/Put/seqid=0 2024-11-13T18:13:56,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742286_1462 (size=12301) 2024-11-13T18:13:56,897 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/7656ba08010849a9a8c8779dc08d202e 2024-11-13T18:13:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/81ef485750064d68bce4e5b93dbc2a5d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/81ef485750064d68bce4e5b93dbc2a5d 2024-11-13T18:13:56,913 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/81ef485750064d68bce4e5b93dbc2a5d, entries=150, sequenceid=260, filesize=30.5 K 2024-11-13T18:13:56,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/486ceced31e7447393637fb3d223d659 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/486ceced31e7447393637fb3d223d659 2024-11-13T18:13:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,922 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/486ceced31e7447393637fb3d223d659, entries=150, sequenceid=260, filesize=12.0 K 2024-11-13T18:13:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:13:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/7656ba08010849a9a8c8779dc08d202e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7656ba08010849a9a8c8779dc08d202e 2024-11-13T18:13:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,933 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7656ba08010849a9a8c8779dc08d202e, entries=150, sequenceid=260, filesize=12.0 K 2024-11-13T18:13:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,934 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for fc0f52f0b3b547094f90472a203572fc in 682ms, sequenceid=260, compaction requested=false 2024-11-13T18:13:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-13T18:13:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-13T18:13:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-13T18:13:56,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1540 sec 2024-11-13T18:13:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.1630 sec 2024-11-13T18:13:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:57,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:13:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:57,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:57,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:57,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:57,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:57,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b619f7621910456398f28912cce2ef0b_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521637193/Put/seqid=0 2024-11-13T18:13:57,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742287_1463 (size=22618) 2024-11-13T18:13:57,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521637311, max timestamp=1731521637346Num uniques 859; 2024-11-13T18:13:57,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:13:57,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521637311, max timestamp=1731521637346Num uniques 859; 2024-11-13T18:13:57,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:13:57,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521637311, max timestamp=1731521637346Num uniques 859; 2024-11-13T18:13:57,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:13:57,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521697389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521697394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521697395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521697397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521697397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521697514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521697525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521697525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521697525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521697527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,713 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:57,717 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b619f7621910456398f28912cce2ef0b_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b619f7621910456398f28912cce2ef0b_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:57,718 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/f96e5f710b4b4793b02e258f772feb5c, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:57,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/f96e5f710b4b4793b02e258f772feb5c is 175, key is test_row_0/A:col10/1731521637193/Put/seqid=0 2024-11-13T18:13:57,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742288_1464 (size=66023) 2024-11-13T18:13:57,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/f96e5f710b4b4793b02e258f772feb5c 2024-11-13T18:13:57,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521697734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521697744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521697744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521697744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/11dd8e9bad60463eb35db58ce4cd3289 is 50, key is test_row_0/B:col10/1731521637193/Put/seqid=0 2024-11-13T18:13:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521697755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:57,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742289_1465 (size=12301) 2024-11-13T18:13:57,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/11dd8e9bad60463eb35db58ce4cd3289 2024-11-13T18:13:57,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/b14790eb338f49788149ff17c7491b82 is 50, key is test_row_0/C:col10/1731521637193/Put/seqid=0 2024-11-13T18:13:57,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742290_1466 (size=12301) 2024-11-13T18:13:57,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/b14790eb338f49788149ff17c7491b82 2024-11-13T18:13:57,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/f96e5f710b4b4793b02e258f772feb5c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f96e5f710b4b4793b02e258f772feb5c 2024-11-13T18:13:57,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f96e5f710b4b4793b02e258f772feb5c, entries=350, sequenceid=273, filesize=64.5 K 2024-11-13T18:13:57,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/11dd8e9bad60463eb35db58ce4cd3289 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/11dd8e9bad60463eb35db58ce4cd3289 2024-11-13T18:13:57,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/11dd8e9bad60463eb35db58ce4cd3289, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T18:13:57,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/b14790eb338f49788149ff17c7491b82 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/b14790eb338f49788149ff17c7491b82 2024-11-13T18:13:57,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/b14790eb338f49788149ff17c7491b82, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T18:13:57,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for fc0f52f0b3b547094f90472a203572fc in 596ms, sequenceid=273, compaction requested=true 2024-11-13T18:13:57,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:57,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:57,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:57,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:57,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:57,873 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:57,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:57,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:13:57,873 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:57,874 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:57,874 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:13:57,874 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:57,874 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/233ad71fe17c4c25a52ba07d9a9ed476, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/486ceced31e7447393637fb3d223d659, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/11dd8e9bad60463eb35db58ce4cd3289] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=36.5 K 2024-11-13T18:13:57,875 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128997 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:57,875 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:13:57,875 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:57,875 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/658a01b5099a4a6d84e0acdab8092205, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/81ef485750064d68bce4e5b93dbc2a5d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f96e5f710b4b4793b02e258f772feb5c] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=126.0 K 2024-11-13T18:13:57,875 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:57,875 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/658a01b5099a4a6d84e0acdab8092205, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/81ef485750064d68bce4e5b93dbc2a5d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f96e5f710b4b4793b02e258f772feb5c] 2024-11-13T18:13:57,875 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 233ad71fe17c4c25a52ba07d9a9ed476, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731521634524 2024-11-13T18:13:57,875 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 658a01b5099a4a6d84e0acdab8092205, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731521634524 2024-11-13T18:13:57,876 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 486ceced31e7447393637fb3d223d659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731521634926 2024-11-13T18:13:57,876 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ef485750064d68bce4e5b93dbc2a5d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731521634926 2024-11-13T18:13:57,876 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 11dd8e9bad60463eb35db58ce4cd3289, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731521637193 2024-11-13T18:13:57,877 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f96e5f710b4b4793b02e258f772feb5c, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731521637193 2024-11-13T18:13:57,889 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:57,902 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:57,903 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/c0bce5c8b51e484cabed36b8588e42ed is 50, key is test_row_0/B:col10/1731521637193/Put/seqid=0 2024-11-13T18:13:57,909 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411139b0826feee744555997588986f95235d_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:57,912 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411139b0826feee744555997588986f95235d_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:57,912 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139b0826feee744555997588986f95235d_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:57,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742291_1467 (size=13017) 2024-11-13T18:13:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:13:57,935 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:13:57,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:13:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-13T18:13:57,939 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:13:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:13:57,939 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:13:57,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:13:57,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742292_1468 (size=4469) 2024-11-13T18:13:58,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:13:58,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:58,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-13T18:13:58,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:58,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:58,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:58,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:58,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:58,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:58,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113880de4562a6349e1bc6de9538184874a_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521637366/Put/seqid=0 2024-11-13T18:13:58,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742293_1469 (size=14994) 2024-11-13T18:13:58,085 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:58,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521698079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,089 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113880de4562a6349e1bc6de9538184874a_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113880de4562a6349e1bc6de9538184874a_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:58,090 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/a304c8cd8fd841f89d98b1ca2ca5e43c, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:58,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/a304c8cd8fd841f89d98b1ca2ca5e43c is 175, key is test_row_0/A:col10/1731521637366/Put/seqid=0 2024-11-13T18:13:58,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:13:58,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742294_1470 (size=39949) 2024-11-13T18:13:58,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:58,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/a304c8cd8fd841f89d98b1ca2ca5e43c 2024-11-13T18:13:58,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521698082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521698082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521698085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521698087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2f4ad2fe0aad4d7783b3f1a82886f077 is 50, key is test_row_0/B:col10/1731521637366/Put/seqid=0 2024-11-13T18:13:58,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742295_1471 (size=12301) 2024-11-13T18:13:58,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2f4ad2fe0aad4d7783b3f1a82886f077 2024-11-13T18:13:58,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/e09556a8b2284717a4db51291b0b2ddb is 50, key is test_row_0/C:col10/1731521637366/Put/seqid=0 2024-11-13T18:13:58,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742296_1472 (size=12301) 2024-11-13T18:13:58,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/e09556a8b2284717a4db51291b0b2ddb 2024-11-13T18:13:58,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/a304c8cd8fd841f89d98b1ca2ca5e43c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/a304c8cd8fd841f89d98b1ca2ca5e43c 2024-11-13T18:13:58,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/a304c8cd8fd841f89d98b1ca2ca5e43c, entries=200, sequenceid=309, filesize=39.0 K 2024-11-13T18:13:58,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2f4ad2fe0aad4d7783b3f1a82886f077 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2f4ad2fe0aad4d7783b3f1a82886f077 2024-11-13T18:13:58,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2f4ad2fe0aad4d7783b3f1a82886f077, entries=150, sequenceid=309, filesize=12.0 K 2024-11-13T18:13:58,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/e09556a8b2284717a4db51291b0b2ddb as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e09556a8b2284717a4db51291b0b2ddb 2024-11-13T18:13:58,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e09556a8b2284717a4db51291b0b2ddb, entries=150, sequenceid=309, filesize=12.0 K 2024-11-13T18:13:58,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for fc0f52f0b3b547094f90472a203572fc in 118ms, sequenceid=309, compaction requested=true 2024-11-13T18:13:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:13:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:13:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T18:13:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:58,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:13:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:13:58,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113635a67e748b640bcb3abc6cddeb496d7_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521638085/Put/seqid=0 2024-11-13T18:13:58,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742297_1473 (size=17534) 2024-11-13T18:13:58,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:13:58,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:58,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,248 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113635a67e748b640bcb3abc6cddeb496d7_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113635a67e748b640bcb3abc6cddeb496d7_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:13:58,255 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/1cba0a2aa6294404a1776b617297dc55, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:58,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/1cba0a2aa6294404a1776b617297dc55 is 175, key is test_row_0/A:col10/1731521638085/Put/seqid=0 2024-11-13T18:13:58,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742298_1474 (size=48639) 2024-11-13T18:13:58,332 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/c0bce5c8b51e484cabed36b8588e42ed as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c0bce5c8b51e484cabed36b8588e42ed 2024-11-13T18:13:58,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521638307, max timestamp=1731521638333Num uniques 859; 2024-11-13T18:13:58,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:58,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521638307, max timestamp=1731521638333Num uniques 859; 2024-11-13T18:13:58,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:58,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521638307, max timestamp=1731521638334Num uniques 859; 2024-11-13T18:13:58,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:13:58,348 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into c0bce5c8b51e484cabed36b8588e42ed(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:58,349 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:58,349 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521637873; duration=0sec 2024-11-13T18:13:58,350 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T18:13:58,350 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:13:58,350 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-13T18:13:58,350 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#393 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:58,351 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/086adaa32e224f498461578ec7ffe708 is 175, key is test_row_0/A:col10/1731521637193/Put/seqid=0 2024-11-13T18:13:58,352 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:13:58,352 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:13:58,352 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. because compaction request was cancelled 2024-11-13T18:13:58,352 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:13:58,352 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:13:58,358 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:13:58,358 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/C is initiating minor compaction (all files) 2024-11-13T18:13:58,358 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/C in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,358 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e9c21d205bac459ba938e0c4a843bbbf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7656ba08010849a9a8c8779dc08d202e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/b14790eb338f49788149ff17c7491b82, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e09556a8b2284717a4db51291b0b2ddb] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=48.5 K 2024-11-13T18:13:58,358 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e9c21d205bac459ba938e0c4a843bbbf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731521634524 2024-11-13T18:13:58,359 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 7656ba08010849a9a8c8779dc08d202e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731521634926 2024-11-13T18:13:58,359 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b14790eb338f49788149ff17c7491b82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731521637193 2024-11-13T18:13:58,360 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e09556a8b2284717a4db51291b0b2ddb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731521637366 2024-11-13T18:13:58,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742299_1475 (size=31971) 2024-11-13T18:13:58,367 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/086adaa32e224f498461578ec7ffe708 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/086adaa32e224f498461578ec7ffe708 2024-11-13T18:13:58,370 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#C#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:58,370 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/a24e76e90f544ad49ac8eadaaf766b42 is 50, key is test_row_0/C:col10/1731521637366/Put/seqid=0 2024-11-13T18:13:58,373 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into 086adaa32e224f498461578ec7ffe708(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:58,373 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:58,373 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521637873; duration=0sec 2024-11-13T18:13:58,373 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:13:58,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:13:58,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:13:58,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-13T18:13:58,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:13:58,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:13:58,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. because compaction request was cancelled 2024-11-13T18:13:58,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:13:58,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:13:58,384 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:13:58,384 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:13:58,384 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. because compaction request was cancelled 2024-11-13T18:13:58,384 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:13:58,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742300_1476 (size=13051) 2024-11-13T18:13:58,395 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/a24e76e90f544ad49ac8eadaaf766b42 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a24e76e90f544ad49ac8eadaaf766b42 2024-11-13T18:13:58,401 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/C of fc0f52f0b3b547094f90472a203572fc into a24e76e90f544ad49ac8eadaaf766b42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:58,401 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:58,401 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/C, priority=12, startTime=1731521638182; duration=0sec 2024-11-13T18:13:58,401 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:58,401 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:13:58,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:13:58,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521698409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521698409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521698409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521698430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521698432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521698534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521698534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521698545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521698554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521698556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:13:58,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:58,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:13:58,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/1cba0a2aa6294404a1776b617297dc55 2024-11-13T18:13:58,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/27cf419bdd4d4fe0bc0a97ade31d9747 is 50, key is test_row_0/B:col10/1731521638085/Put/seqid=0 2024-11-13T18:13:58,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742301_1477 (size=12301) 2024-11-13T18:13:58,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/27cf419bdd4d4fe0bc0a97ade31d9747 2024-11-13T18:13:58,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:13:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:13:58,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/2985918bbb6e4eed846aa2c49cb4bbea is 50, key is test_row_0/C:col10/1731521638085/Put/seqid=0 2024-11-13T18:13:58,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742302_1478 (size=12301) 2024-11-13T18:13:58,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521698757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521698757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/2985918bbb6e4eed846aa2c49cb4bbea 2024-11-13T18:13:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521698765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521698784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521698785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:58,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:58,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:58,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/1cba0a2aa6294404a1776b617297dc55 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/1cba0a2aa6294404a1776b617297dc55 2024-11-13T18:13:58,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/1cba0a2aa6294404a1776b617297dc55, entries=250, sequenceid=320, filesize=47.5 K 2024-11-13T18:13:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/27cf419bdd4d4fe0bc0a97ade31d9747 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/27cf419bdd4d4fe0bc0a97ade31d9747 2024-11-13T18:13:58,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/27cf419bdd4d4fe0bc0a97ade31d9747, entries=150, sequenceid=320, filesize=12.0 K 2024-11-13T18:13:58,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/2985918bbb6e4eed846aa2c49cb4bbea as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2985918bbb6e4eed846aa2c49cb4bbea 2024-11-13T18:13:58,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2985918bbb6e4eed846aa2c49cb4bbea, entries=150, sequenceid=320, filesize=12.0 K 2024-11-13T18:13:58,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fc0f52f0b3b547094f90472a203572fc in 631ms, sequenceid=320, compaction requested=true 2024-11-13T18:13:58,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:58,841 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:58,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:13:58,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:58,841 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:13:58,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:13:58,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:58,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:13:58,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:58,842 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120559 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:58,842 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:13:58,842 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,842 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/086adaa32e224f498461578ec7ffe708, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/a304c8cd8fd841f89d98b1ca2ca5e43c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/1cba0a2aa6294404a1776b617297dc55] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=117.7 K 2024-11-13T18:13:58,842 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,842 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/086adaa32e224f498461578ec7ffe708, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/a304c8cd8fd841f89d98b1ca2ca5e43c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/1cba0a2aa6294404a1776b617297dc55] 2024-11-13T18:13:58,843 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 086adaa32e224f498461578ec7ffe708, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731521637193 2024-11-13T18:13:58,843 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a304c8cd8fd841f89d98b1ca2ca5e43c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731521637366 2024-11-13T18:13:58,844 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:13:58,844 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:13:58,844 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,844 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c0bce5c8b51e484cabed36b8588e42ed, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2f4ad2fe0aad4d7783b3f1a82886f077, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/27cf419bdd4d4fe0bc0a97ade31d9747] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=36.7 K 2024-11-13T18:13:58,844 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cba0a2aa6294404a1776b617297dc55, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731521638078 2024-11-13T18:13:58,845 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c0bce5c8b51e484cabed36b8588e42ed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731521637193 2024-11-13T18:13:58,845 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4ad2fe0aad4d7783b3f1a82886f077, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731521637366 2024-11-13T18:13:58,846 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 27cf419bdd4d4fe0bc0a97ade31d9747, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731521638085 2024-11-13T18:13:58,853 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:58,858 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:58,859 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/94b8079ddec0421fa794dd604a24536e is 50, key is test_row_0/B:col10/1731521638085/Put/seqid=0 2024-11-13T18:13:58,863 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ccd51d38075a42c59af24090c033667c_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:58,869 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ccd51d38075a42c59af24090c033667c_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:58,869 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ccd51d38075a42c59af24090c033667c_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:58,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:13:58,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:13:58,893 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-13T18:13:58,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:13:58,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:58,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:13:58,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:58,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:13:58,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:13:58,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742303_1479 (size=13119) 2024-11-13T18:13:58,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742304_1480 (size=4469) 2024-11-13T18:13:58,937 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#402 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:13:58,938 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/10166af818d14027b9960148fc5a799c is 175, key is test_row_0/A:col10/1731521638085/Put/seqid=0 2024-11-13T18:13:58,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c2d6e94a417e4982ababc5bca58a2672_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521638403/Put/seqid=0 2024-11-13T18:13:58,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742305_1481 (size=32073) 2024-11-13T18:13:59,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742306_1482 (size=12454) 2024-11-13T18:13:59,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:13:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:13:59,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521699118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521699120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521699120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521699121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521699121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521699234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521699244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521699244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521699245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521699245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,319 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/94b8079ddec0421fa794dd604a24536e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/94b8079ddec0421fa794dd604a24536e 2024-11-13T18:13:59,324 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into 94b8079ddec0421fa794dd604a24536e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:59,324 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:59,324 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521638841; duration=0sec 2024-11-13T18:13:59,324 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:13:59,324 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:13:59,324 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:13:59,325 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:13:59,325 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:13:59,325 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. because compaction request was cancelled 2024-11-13T18:13:59,325 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:13:59,384 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/10166af818d14027b9960148fc5a799c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/10166af818d14027b9960148fc5a799c 2024-11-13T18:13:59,392 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into 10166af818d14027b9960148fc5a799c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:13:59,392 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:13:59,392 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521638841; duration=0sec 2024-11-13T18:13:59,392 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:13:59,392 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:13:59,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:13:59,411 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c2d6e94a417e4982ababc5bca58a2672_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c2d6e94a417e4982ababc5bca58a2672_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:13:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/e358de78d20c433084c36c328309c1d9, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:13:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/e358de78d20c433084c36c328309c1d9 is 175, key is test_row_0/A:col10/1731521638403/Put/seqid=0 2024-11-13T18:13:59,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742307_1483 (size=31255) 2024-11-13T18:13:59,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521699444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521699464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521699464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521699469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521699469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521699764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521699784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521699784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521699794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:13:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521699795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:13:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:13:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:13:59,836 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/e358de78d20c433084c36c328309c1d9 2024-11-13T18:13:59,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/88206899ef314701917750175fc87a69 is 50, key is test_row_0/B:col10/1731521638403/Put/seqid=0 2024-11-13T18:13:59,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742308_1484 (size=12301) 2024-11-13T18:14:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:14:00,262 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/88206899ef314701917750175fc87a69 2024-11-13T18:14:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/2b44ae4555c740d49bebb6453590c18c is 50, key is test_row_0/C:col10/1731521638403/Put/seqid=0 2024-11-13T18:14:00,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742309_1485 (size=12301) 2024-11-13T18:14:00,284 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/2b44ae4555c740d49bebb6453590c18c 2024-11-13T18:14:00,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/e358de78d20c433084c36c328309c1d9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/e358de78d20c433084c36c328309c1d9 2024-11-13T18:14:00,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521700294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:00,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:00,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:00,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521700304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:00,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:00,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521700304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:00,315 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/e358de78d20c433084c36c328309c1d9, entries=150, sequenceid=359, filesize=30.5 K 2024-11-13T18:14:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/88206899ef314701917750175fc87a69 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/88206899ef314701917750175fc87a69 2024-11-13T18:14:00,320 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/88206899ef314701917750175fc87a69, entries=150, sequenceid=359, filesize=12.0 K 2024-11-13T18:14:00,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/2b44ae4555c740d49bebb6453590c18c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2b44ae4555c740d49bebb6453590c18c 2024-11-13T18:14:00,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521700314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:00,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521700325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:00,338 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2b44ae4555c740d49bebb6453590c18c, entries=150, sequenceid=359, filesize=12.0 K 2024-11-13T18:14:00,340 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for fc0f52f0b3b547094f90472a203572fc in 1447ms, sequenceid=359, compaction requested=true 2024-11-13T18:14:00,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:00,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:00,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-13T18:14:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-13T18:14:00,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-13T18:14:00,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4050 sec 2024-11-13T18:14:00,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.4150 sec 2024-11-13T18:14:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:01,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:14:01,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:14:01,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:01,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:14:01,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:01,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:14:01,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:01,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111353dc61ebac7e44b799d7fd491be2a441_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521639118/Put/seqid=0 2024-11-13T18:14:01,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742310_1486 (size=14994) 2024-11-13T18:14:01,360 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:01,370 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111353dc61ebac7e44b799d7fd491be2a441_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111353dc61ebac7e44b799d7fd491be2a441_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:01,371 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/6c4b81df5c1741ff8d0c0ec62768e9ff, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:01,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/6c4b81df5c1741ff8d0c0ec62768e9ff is 175, key is test_row_0/A:col10/1731521639118/Put/seqid=0 2024-11-13T18:14:01,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742311_1487 (size=39949) 2024-11-13T18:14:01,376 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/6c4b81df5c1741ff8d0c0ec62768e9ff 2024-11-13T18:14:01,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/fdc59f4ebe8f429fbdafabe2a0643688 is 50, key is test_row_0/B:col10/1731521639118/Put/seqid=0 2024-11-13T18:14:01,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742312_1488 (size=12301) 2024-11-13T18:14:01,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521701376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521701377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521701378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521701379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521701391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521701494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521701504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521701504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521701504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521701506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521701704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521701725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521701725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521701728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:01,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521701729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:01,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/fdc59f4ebe8f429fbdafabe2a0643688 2024-11-13T18:14:01,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/c9cb765eabdf4549bb6e0755515e26e3 is 50, key is test_row_0/C:col10/1731521639118/Put/seqid=0 2024-11-13T18:14:01,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742313_1489 (size=12301) 2024-11-13T18:14:02,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521702024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521702044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521702044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521702044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521702044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:14:02,104 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:02,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-13T18:14:02,112 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:14:02,113 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:02,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:02,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/c9cb765eabdf4549bb6e0755515e26e3 2024-11-13T18:14:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:14:02,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/6c4b81df5c1741ff8d0c0ec62768e9ff as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6c4b81df5c1741ff8d0c0ec62768e9ff 2024-11-13T18:14:02,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6c4b81df5c1741ff8d0c0ec62768e9ff, entries=200, sequenceid=374, filesize=39.0 K 2024-11-13T18:14:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/fdc59f4ebe8f429fbdafabe2a0643688 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/fdc59f4ebe8f429fbdafabe2a0643688 2024-11-13T18:14:02,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/fdc59f4ebe8f429fbdafabe2a0643688, entries=150, sequenceid=374, filesize=12.0 K 2024-11-13T18:14:02,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/c9cb765eabdf4549bb6e0755515e26e3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/c9cb765eabdf4549bb6e0755515e26e3 2024-11-13T18:14:02,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/c9cb765eabdf4549bb6e0755515e26e3, entries=150, sequenceid=374, filesize=12.0 K 2024-11-13T18:14:02,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fc0f52f0b3b547094f90472a203572fc in 915ms, sequenceid=374, compaction requested=true 2024-11-13T18:14:02,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:02,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:02,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:02,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:02,249 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:02,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:02,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:02,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:02,249 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:02,250 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:02,250 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:02,250 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:14:02,250 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:14:02,250 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:02,250 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:02,250 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/94b8079ddec0421fa794dd604a24536e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/88206899ef314701917750175fc87a69, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/fdc59f4ebe8f429fbdafabe2a0643688] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=36.8 K 2024-11-13T18:14:02,250 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/10166af818d14027b9960148fc5a799c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/e358de78d20c433084c36c328309c1d9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6c4b81df5c1741ff8d0c0ec62768e9ff] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=100.9 K 2024-11-13T18:14:02,250 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:02,250 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/10166af818d14027b9960148fc5a799c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/e358de78d20c433084c36c328309c1d9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6c4b81df5c1741ff8d0c0ec62768e9ff] 2024-11-13T18:14:02,252 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b8079ddec0421fa794dd604a24536e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731521638085 2024-11-13T18:14:02,252 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10166af818d14027b9960148fc5a799c, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731521638085 2024-11-13T18:14:02,253 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 88206899ef314701917750175fc87a69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731521638366 2024-11-13T18:14:02,253 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e358de78d20c433084c36c328309c1d9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731521638366 2024-11-13T18:14:02,253 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc59f4ebe8f429fbdafabe2a0643688, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521639118 2024-11-13T18:14:02,253 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c4b81df5c1741ff8d0c0ec62768e9ff, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521639116 2024-11-13T18:14:02,263 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:02,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T18:14:02,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:02,265 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/39b9a29b41c14aafa46e9edfa8aee489 is 50, key is test_row_0/B:col10/1731521639118/Put/seqid=0 2024-11-13T18:14:02,265 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:14:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:14:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:14:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:14:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:02,278 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:02,293 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411130359d0f062034220b21e228fd7177cc3_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:02,296 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411130359d0f062034220b21e228fd7177cc3_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:02,296 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130359d0f062034220b21e228fd7177cc3_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113709460571a66402a9642312b9847a6cf_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521641375/Put/seqid=0 2024-11-13T18:14:02,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742314_1490 (size=13221) 2024-11-13T18:14:02,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742315_1491 (size=4469) 2024-11-13T18:14:02,337 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#411 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:02,337 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/9d2c7a053ee74e3c935d2ce1f7ba89a5 is 175, key is test_row_0/A:col10/1731521639118/Put/seqid=0 2024-11-13T18:14:02,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742316_1492 (size=12454) 2024-11-13T18:14:02,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:02,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742317_1493 (size=32175) 2024-11-13T18:14:02,347 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113709460571a66402a9642312b9847a6cf_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113709460571a66402a9642312b9847a6cf_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:02,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/3d554e31a359477dbb81f67cdaffc5ef, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/3d554e31a359477dbb81f67cdaffc5ef is 175, key is test_row_0/A:col10/1731521641375/Put/seqid=0 2024-11-13T18:14:02,357 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/9d2c7a053ee74e3c935d2ce1f7ba89a5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/9d2c7a053ee74e3c935d2ce1f7ba89a5 2024-11-13T18:14:02,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742318_1494 (size=31255) 2024-11-13T18:14:02,359 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/3d554e31a359477dbb81f67cdaffc5ef 2024-11-13T18:14:02,364 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into 9d2c7a053ee74e3c935d2ce1f7ba89a5(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:02,364 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:02,364 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521642248; duration=0sec 2024-11-13T18:14:02,364 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:02,365 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:14:02,365 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:02,367 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:02,367 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/C is initiating minor compaction (all files) 2024-11-13T18:14:02,367 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/C in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:02,368 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a24e76e90f544ad49ac8eadaaf766b42, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2985918bbb6e4eed846aa2c49cb4bbea, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2b44ae4555c740d49bebb6453590c18c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/c9cb765eabdf4549bb6e0755515e26e3] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=48.8 K 2024-11-13T18:14:02,368 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a24e76e90f544ad49ac8eadaaf766b42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731521637366 2024-11-13T18:14:02,368 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2985918bbb6e4eed846aa2c49cb4bbea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731521638085 2024-11-13T18:14:02,369 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b44ae4555c740d49bebb6453590c18c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731521638366 2024-11-13T18:14:02,369 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9cb765eabdf4549bb6e0755515e26e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521639118 2024-11-13T18:14:02,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/0d2e2da14e9e45899284bcca1f2af7af is 50, key is test_row_0/B:col10/1731521641375/Put/seqid=0 2024-11-13T18:14:02,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742319_1495 (size=12301) 2024-11-13T18:14:02,392 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#C#compaction#414 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:02,400 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/f029ff2dd1014424888bb1607ff9a57d is 50, key is test_row_0/C:col10/1731521639118/Put/seqid=0 2024-11-13T18:14:02,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742320_1496 (size=13187) 2024-11-13T18:14:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:14:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:02,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:02,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521702584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521702586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521702587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521702588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521702588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521702694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521702694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,723 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/39b9a29b41c14aafa46e9edfa8aee489 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/39b9a29b41c14aafa46e9edfa8aee489 2024-11-13T18:14:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521702714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521702714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521702714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,727 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into 39b9a29b41c14aafa46e9edfa8aee489(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:02,727 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:02,727 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521642249; duration=0sec 2024-11-13T18:14:02,727 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:02,727 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:14:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:14:02,791 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/0d2e2da14e9e45899284bcca1f2af7af 2024-11-13T18:14:02,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/1c76cec9bf774448957be46d7bc0a471 is 50, key is test_row_0/C:col10/1731521641375/Put/seqid=0 2024-11-13T18:14:02,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742321_1497 (size=12301) 2024-11-13T18:14:02,822 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/f029ff2dd1014424888bb1607ff9a57d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f029ff2dd1014424888bb1607ff9a57d 2024-11-13T18:14:02,826 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/C of fc0f52f0b3b547094f90472a203572fc into f029ff2dd1014424888bb1607ff9a57d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:02,827 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:02,827 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/C, priority=12, startTime=1731521642249; duration=0sec 2024-11-13T18:14:02,827 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:02,827 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:14:02,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521702914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521702914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521702934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521702934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521702934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,204 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/1c76cec9bf774448957be46d7bc0a471 2024-11-13T18:14:03,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/3d554e31a359477dbb81f67cdaffc5ef as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3d554e31a359477dbb81f67cdaffc5ef 2024-11-13T18:14:03,211 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3d554e31a359477dbb81f67cdaffc5ef, entries=150, sequenceid=393, filesize=30.5 K 2024-11-13T18:14:03,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/0d2e2da14e9e45899284bcca1f2af7af as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/0d2e2da14e9e45899284bcca1f2af7af 2024-11-13T18:14:03,217 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/0d2e2da14e9e45899284bcca1f2af7af, entries=150, sequenceid=393, filesize=12.0 K 2024-11-13T18:14:03,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/1c76cec9bf774448957be46d7bc0a471 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c76cec9bf774448957be46d7bc0a471 2024-11-13T18:14:03,221 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c76cec9bf774448957be46d7bc0a471, entries=150, sequenceid=393, filesize=12.0 K 2024-11-13T18:14:03,222 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fc0f52f0b3b547094f90472a203572fc in 957ms, sequenceid=393, compaction requested=false 2024-11-13T18:14:03,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:03,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:03,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-13T18:14:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-13T18:14:03,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-13T18:14:03,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1100 sec 2024-11-13T18:14:03,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.1170 sec 2024-11-13T18:14:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:03,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T18:14:03,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:14:03,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:03,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:14:03,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:03,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:14:03,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:03,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111342a0c15bc352483ea765257fb9c8cde1_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521642582/Put/seqid=0 2024-11-13T18:14:03,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742322_1498 (size=14994) 2024-11-13T18:14:03,240 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:03,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111342a0c15bc352483ea765257fb9c8cde1_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111342a0c15bc352483ea765257fb9c8cde1_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:03,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/03d7ebb3824544f88d00d9f037f39a88, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:03,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/03d7ebb3824544f88d00d9f037f39a88 is 175, key is test_row_0/A:col10/1731521642582/Put/seqid=0 2024-11-13T18:14:03,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742323_1499 (size=39949) 2024-11-13T18:14:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:14:03,254 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:03,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-13T18:14:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:14:03,257 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:03,257 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:03,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:03,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521703269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521703270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521703274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521703277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521703284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:14:03,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521703394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521703394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521703394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521703394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T18:14:03,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:03,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:03,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:03,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521703404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T18:14:03,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:03,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:03,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:03,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:14:03,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521703604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521703604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521703604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521703604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521703624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,650 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/03d7ebb3824544f88d00d9f037f39a88 2024-11-13T18:14:03,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2a13be6b511648a8b470852a744a6197 is 50, key is test_row_0/B:col10/1731521642582/Put/seqid=0 2024-11-13T18:14:03,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742324_1500 (size=12301) 2024-11-13T18:14:03,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T18:14:03,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:03,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:03,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:03,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T18:14:03,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:03,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:03,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:03,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:14:03,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521703914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521703914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521703924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521703925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:03,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521703944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T18:14:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:04,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:04,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2a13be6b511648a8b470852a744a6197 2024-11-13T18:14:04,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/7496cab9e041410dabd7e9ff13ce423c is 50, key is test_row_0/C:col10/1731521642582/Put/seqid=0 2024-11-13T18:14:04,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742325_1501 (size=12301) 2024-11-13T18:14:04,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/7496cab9e041410dabd7e9ff13ce423c 2024-11-13T18:14:04,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/03d7ebb3824544f88d00d9f037f39a88 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/03d7ebb3824544f88d00d9f037f39a88 2024-11-13T18:14:04,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/03d7ebb3824544f88d00d9f037f39a88, entries=200, sequenceid=410, filesize=39.0 K 2024-11-13T18:14:04,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/2a13be6b511648a8b470852a744a6197 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2a13be6b511648a8b470852a744a6197 2024-11-13T18:14:04,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2a13be6b511648a8b470852a744a6197, entries=150, sequenceid=410, filesize=12.0 K 2024-11-13T18:14:04,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/7496cab9e041410dabd7e9ff13ce423c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7496cab9e041410dabd7e9ff13ce423c 2024-11-13T18:14:04,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7496cab9e041410dabd7e9ff13ce423c, entries=150, sequenceid=410, filesize=12.0 K 2024-11-13T18:14:04,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fc0f52f0b3b547094f90472a203572fc in 887ms, sequenceid=410, compaction requested=true 2024-11-13T18:14:04,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:04,115 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:04,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:04,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:04,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:04,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:04,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:04,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:04,116 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:04,117 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:04,117 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:14:04,117 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:04,117 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:04,117 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/9d2c7a053ee74e3c935d2ce1f7ba89a5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3d554e31a359477dbb81f67cdaffc5ef, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/03d7ebb3824544f88d00d9f037f39a88] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=101.0 K 2024-11-13T18:14:04,117 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:14:04,117 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:04,117 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/9d2c7a053ee74e3c935d2ce1f7ba89a5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3d554e31a359477dbb81f67cdaffc5ef, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/03d7ebb3824544f88d00d9f037f39a88] 2024-11-13T18:14:04,117 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:04,118 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/39b9a29b41c14aafa46e9edfa8aee489, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/0d2e2da14e9e45899284bcca1f2af7af, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2a13be6b511648a8b470852a744a6197] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=36.9 K 2024-11-13T18:14:04,119 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d2c7a053ee74e3c935d2ce1f7ba89a5, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521639118 2024-11-13T18:14:04,119 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b9a29b41c14aafa46e9edfa8aee489, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521639118 2024-11-13T18:14:04,119 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d554e31a359477dbb81f67cdaffc5ef, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731521641375 2024-11-13T18:14:04,121 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2e2da14e9e45899284bcca1f2af7af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731521641375 2024-11-13T18:14:04,122 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03d7ebb3824544f88d00d9f037f39a88, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731521642582 2024-11-13T18:14:04,122 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a13be6b511648a8b470852a744a6197, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731521642582 2024-11-13T18:14:04,142 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:04,142 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/e3e8e07d25254940926971d592d5708c is 50, key is test_row_0/B:col10/1731521642582/Put/seqid=0 2024-11-13T18:14:04,143 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:04,147 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113d3075a9499c74fefa31534c8f0a6f5d1_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:04,150 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113d3075a9499c74fefa31534c8f0a6f5d1_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:04,157 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d3075a9499c74fefa31534c8f0a6f5d1_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:04,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742326_1502 (size=13323) 2024-11-13T18:14:04,182 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/e3e8e07d25254940926971d592d5708c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/e3e8e07d25254940926971d592d5708c 2024-11-13T18:14:04,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T18:14:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:04,183 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:14:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:14:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:14:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:14:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:04,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742327_1503 (size=4469) 2024-11-13T18:14:04,188 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into e3e8e07d25254940926971d592d5708c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:04,188 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:04,188 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521644115; duration=0sec 2024-11-13T18:14:04,189 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:04,189 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:14:04,189 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:04,190 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#420 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:04,191 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b847a59ef3ca425589246776108972d4 is 175, key is test_row_0/A:col10/1731521642582/Put/seqid=0 2024-11-13T18:14:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111302af861295c14d3cad9e254f916319c3_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521643267/Put/seqid=0 2024-11-13T18:14:04,200 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:04,200 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/C is initiating minor compaction (all files) 2024-11-13T18:14:04,200 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/C in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:04,200 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f029ff2dd1014424888bb1607ff9a57d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c76cec9bf774448957be46d7bc0a471, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7496cab9e041410dabd7e9ff13ce423c] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=36.9 K 2024-11-13T18:14:04,204 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f029ff2dd1014424888bb1607ff9a57d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731521639118 2024-11-13T18:14:04,209 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c76cec9bf774448957be46d7bc0a471, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731521641375 2024-11-13T18:14:04,213 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 7496cab9e041410dabd7e9ff13ce423c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731521642582 2024-11-13T18:14:04,221 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#C#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:04,222 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/ac0b438cc4da4367b4de80a3cc7f040c is 50, key is test_row_0/C:col10/1731521642582/Put/seqid=0 2024-11-13T18:14:04,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742329_1505 (size=12454) 2024-11-13T18:14:04,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742328_1504 (size=32277) 2024-11-13T18:14:04,236 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/b847a59ef3ca425589246776108972d4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b847a59ef3ca425589246776108972d4 2024-11-13T18:14:04,242 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into b847a59ef3ca425589246776108972d4(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:04,242 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:04,242 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521644115; duration=0sec 2024-11-13T18:14:04,242 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:04,242 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:14:04,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742330_1506 (size=13289) 2024-11-13T18:14:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:14:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:04,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521704470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521704471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521704528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521704528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521704529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:04,632 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111302af861295c14d3cad9e254f916319c3_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302af861295c14d3cad9e254f916319c3_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:04,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/8a87973c68d14a238f461d6eb0d827eb, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:04,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/8a87973c68d14a238f461d6eb0d827eb is 175, key is test_row_0/A:col10/1731521643267/Put/seqid=0 2024-11-13T18:14:04,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742331_1507 (size=31255) 2024-11-13T18:14:04,643 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=428, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/8a87973c68d14a238f461d6eb0d827eb 2024-11-13T18:14:04,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521704634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521704634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/62fa0ac507c046d89aea521c1340032b is 50, key is test_row_0/B:col10/1731521643267/Put/seqid=0 2024-11-13T18:14:04,659 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/ac0b438cc4da4367b4de80a3cc7f040c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ac0b438cc4da4367b4de80a3cc7f040c 2024-11-13T18:14:04,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521704655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521704655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521704657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,666 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/C of fc0f52f0b3b547094f90472a203572fc into ac0b438cc4da4367b4de80a3cc7f040c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:04,666 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:04,666 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/C, priority=13, startTime=1731521644115; duration=0sec 2024-11-13T18:14:04,666 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:04,666 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:14:04,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742332_1508 (size=12301) 2024-11-13T18:14:04,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521704854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521704854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521704874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521704875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521704876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:04,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:04,984 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:14:05,074 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/62fa0ac507c046d89aea521c1340032b 2024-11-13T18:14:05,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/a54f549a8be14a80bdada488d7fef13e is 50, key is test_row_0/C:col10/1731521643267/Put/seqid=0 2024-11-13T18:14:05,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742333_1509 (size=12301) 2024-11-13T18:14:05,124 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/a54f549a8be14a80bdada488d7fef13e 2024-11-13T18:14:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/8a87973c68d14a238f461d6eb0d827eb as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8a87973c68d14a238f461d6eb0d827eb 2024-11-13T18:14:05,139 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8a87973c68d14a238f461d6eb0d827eb, entries=150, sequenceid=428, filesize=30.5 K 2024-11-13T18:14:05,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/62fa0ac507c046d89aea521c1340032b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/62fa0ac507c046d89aea521c1340032b 2024-11-13T18:14:05,150 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/62fa0ac507c046d89aea521c1340032b, entries=150, sequenceid=428, filesize=12.0 K 2024-11-13T18:14:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/a54f549a8be14a80bdada488d7fef13e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a54f549a8be14a80bdada488d7fef13e 2024-11-13T18:14:05,155 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a54f549a8be14a80bdada488d7fef13e, entries=150, sequenceid=428, filesize=12.0 K 2024-11-13T18:14:05,156 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fc0f52f0b3b547094f90472a203572fc in 973ms, sequenceid=428, compaction requested=false 2024-11-13T18:14:05,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:05,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-13T18:14:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-13T18:14:05,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-13T18:14:05,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9100 sec 2024-11-13T18:14:05,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.9170 sec 2024-11-13T18:14:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:05,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:14:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:14:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:14:05,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:05,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:14:05,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:05,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e799704015da49d493d20085046ddbc7_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521644494/Put/seqid=0 2024-11-13T18:14:05,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742334_1510 (size=12454) 2024-11-13T18:14:05,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521705216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521705229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521705230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521705233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521705234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521705347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521705354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521705356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521705356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521705357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:14:05,414 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:05,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-13T18:14:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:14:05,417 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:05,418 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:05,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:14:05,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:14:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:05,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:05,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:05,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521705564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521705573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521705574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521705574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521705574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,604 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:05,608 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e799704015da49d493d20085046ddbc7_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e799704015da49d493d20085046ddbc7_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:05,609 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/3beccb25477f45abbaac4aa1d3061dbe, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:05,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/3beccb25477f45abbaac4aa1d3061dbe is 175, key is test_row_0/A:col10/1731521644494/Put/seqid=0 2024-11-13T18:14:05,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742335_1511 (size=31255) 2024-11-13T18:14:05,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:14:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:05,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:14:05,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:14:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:05,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:05,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:05,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521705885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521705894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521705894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521705894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521705894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:06,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:14:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:06,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,040 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=447, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/3beccb25477f45abbaac4aa1d3061dbe 2024-11-13T18:14:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:14:06,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/a3d4fc0e57ac494685c82f51fc5b0d55 is 50, key is test_row_0/B:col10/1731521644494/Put/seqid=0 2024-11-13T18:14:06,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742336_1512 (size=12301) 2024-11-13T18:14:06,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:14:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,199 INFO [Thread-1730 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1730. 2024-11-13T18:14:06,199 DEBUG [Thread-1730 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:06,199 DEBUG [Thread-1730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:06,199 DEBUG [Thread-1730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:06,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:06,207 INFO [Thread-1734 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1734. 2024-11-13T18:14:06,207 DEBUG [Thread-1734 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:06,207 DEBUG [Thread-1734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:06,207 DEBUG [Thread-1734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:06,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:06,214 INFO [Thread-1732 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1732. 2024-11-13T18:14:06,214 DEBUG [Thread-1732 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:06,214 DEBUG [Thread-1732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:06,214 DEBUG [Thread-1732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:06,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:06,216 INFO [Thread-1736 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1736. 2024-11-13T18:14:06,216 DEBUG [Thread-1736 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:06,216 DEBUG [Thread-1736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:06,216 DEBUG [Thread-1736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:06,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:06,219 INFO [Thread-1738 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1738. 2024-11-13T18:14:06,219 DEBUG [Thread-1738 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:06,220 DEBUG [Thread-1738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:06,220 DEBUG [Thread-1738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:06,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:06,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:14:06,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:06,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:06,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:06,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55208 deadline: 1731521706404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55176 deadline: 1731521706404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55184 deadline: 1731521706404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:06,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55224 deadline: 1731521706404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:06,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55254 deadline: 1731521706414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:06,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/a3d4fc0e57ac494685c82f51fc5b0d55 2024-11-13T18:14:06,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/0f8a3380909a422689de38983c14f806 is 50, key is test_row_0/C:col10/1731521644494/Put/seqid=0 2024-11-13T18:14:06,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742337_1513 (size=12301) 2024-11-13T18:14:06,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:14:06,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:06,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:14:06,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:14:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:06,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:14:06,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:06,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:06,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/0f8a3380909a422689de38983c14f806 2024-11-13T18:14:06,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/3beccb25477f45abbaac4aa1d3061dbe as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3beccb25477f45abbaac4aa1d3061dbe 2024-11-13T18:14:06,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3beccb25477f45abbaac4aa1d3061dbe, entries=150, sequenceid=447, filesize=30.5 K 2024-11-13T18:14:06,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/a3d4fc0e57ac494685c82f51fc5b0d55 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a3d4fc0e57ac494685c82f51fc5b0d55 2024-11-13T18:14:06,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a3d4fc0e57ac494685c82f51fc5b0d55, entries=150, sequenceid=447, filesize=12.0 K 2024-11-13T18:14:06,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/0f8a3380909a422689de38983c14f806 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/0f8a3380909a422689de38983c14f806 2024-11-13T18:14:06,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/0f8a3380909a422689de38983c14f806, entries=150, sequenceid=447, filesize=12.0 K 2024-11-13T18:14:06,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for fc0f52f0b3b547094f90472a203572fc in 1720ms, sequenceid=447, compaction requested=true 2024-11-13T18:14:06,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:06,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:06,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:06,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:06,902 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:06,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:06,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc0f52f0b3b547094f90472a203572fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:06,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:06,902 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:06,903 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:06,903 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:06,903 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/A is initiating minor compaction (all files) 2024-11-13T18:14:06,903 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/B is initiating minor compaction (all files) 2024-11-13T18:14:06,904 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/A in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,904 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/B in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,904 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b847a59ef3ca425589246776108972d4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8a87973c68d14a238f461d6eb0d827eb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3beccb25477f45abbaac4aa1d3061dbe] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=92.6 K 2024-11-13T18:14:06,904 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,904 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b847a59ef3ca425589246776108972d4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8a87973c68d14a238f461d6eb0d827eb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3beccb25477f45abbaac4aa1d3061dbe] 2024-11-13T18:14:06,904 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/e3e8e07d25254940926971d592d5708c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/62fa0ac507c046d89aea521c1340032b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a3d4fc0e57ac494685c82f51fc5b0d55] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=37.0 K 2024-11-13T18:14:06,904 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b847a59ef3ca425589246776108972d4, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731521642582 2024-11-13T18:14:06,904 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3e8e07d25254940926971d592d5708c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731521642582 2024-11-13T18:14:06,904 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a87973c68d14a238f461d6eb0d827eb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731521643262 2024-11-13T18:14:06,904 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62fa0ac507c046d89aea521c1340032b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731521643262 2024-11-13T18:14:06,905 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 3beccb25477f45abbaac4aa1d3061dbe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731521644470 2024-11-13T18:14:06,905 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d4fc0e57ac494685c82f51fc5b0d55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731521644470 2024-11-13T18:14:06,910 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:06,911 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411132ed00e6b02fd4a34ab68d4bc821d46ef_fc0f52f0b3b547094f90472a203572fc store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:06,912 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:06,913 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/a5ddf4c7298b4a75935fc244e07a5811 is 50, key is test_row_0/B:col10/1731521644494/Put/seqid=0 2024-11-13T18:14:06,915 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411132ed00e6b02fd4a34ab68d4bc821d46ef_fc0f52f0b3b547094f90472a203572fc, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:06,916 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132ed00e6b02fd4a34ab68d4bc821d46ef_fc0f52f0b3b547094f90472a203572fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:06,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742338_1514 (size=13425) 2024-11-13T18:14:06,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742339_1515 (size=4469) 2024-11-13T18:14:06,922 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#A#compaction#428 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:06,922 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/a5ddf4c7298b4a75935fc244e07a5811 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a5ddf4c7298b4a75935fc244e07a5811 2024-11-13T18:14:06,922 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/0fb5d76cefa24a5dae6fb5ee1cfc38bc is 175, key is test_row_0/A:col10/1731521644494/Put/seqid=0 2024-11-13T18:14:06,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742340_1516 (size=32379) 2024-11-13T18:14:06,927 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/B of fc0f52f0b3b547094f90472a203572fc into a5ddf4c7298b4a75935fc244e07a5811(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:06,927 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:06,927 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/B, priority=13, startTime=1731521646902; duration=0sec 2024-11-13T18:14:06,927 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:06,927 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:B 2024-11-13T18:14:06,928 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:06,928 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:06,928 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): fc0f52f0b3b547094f90472a203572fc/C is initiating minor compaction (all files) 2024-11-13T18:14:06,928 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc0f52f0b3b547094f90472a203572fc/C in TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,928 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ac0b438cc4da4367b4de80a3cc7f040c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a54f549a8be14a80bdada488d7fef13e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/0f8a3380909a422689de38983c14f806] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp, totalSize=37.0 K 2024-11-13T18:14:06,929 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac0b438cc4da4367b4de80a3cc7f040c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731521642582 2024-11-13T18:14:06,929 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a54f549a8be14a80bdada488d7fef13e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731521643262 2024-11-13T18:14:06,929 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f8a3380909a422689de38983c14f806, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731521644470 2024-11-13T18:14:06,936 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc0f52f0b3b547094f90472a203572fc#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:06,936 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/ccbc2beecd6045a3b18733ae08a98fa0 is 50, key is test_row_0/C:col10/1731521644494/Put/seqid=0 2024-11-13T18:14:06,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742341_1517 (size=13391) 2024-11-13T18:14:06,955 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/ccbc2beecd6045a3b18733ae08a98fa0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ccbc2beecd6045a3b18733ae08a98fa0 2024-11-13T18:14:06,959 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/C of fc0f52f0b3b547094f90472a203572fc into ccbc2beecd6045a3b18733ae08a98fa0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:06,959 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:06,959 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/C, priority=13, startTime=1731521646902; duration=0sec 2024-11-13T18:14:06,960 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:06,960 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:C 2024-11-13T18:14:06,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:14:06,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:06,960 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:14:06,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:14:06,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:06,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:14:06,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:06,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:14:06,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111302e3cb9c310c4fe484b92eb2e92d01bd_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521645214/Put/seqid=0 2024-11-13T18:14:06,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742342_1518 (size=12454) 2024-11-13T18:14:07,330 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/0fb5d76cefa24a5dae6fb5ee1cfc38bc as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/0fb5d76cefa24a5dae6fb5ee1cfc38bc 2024-11-13T18:14:07,334 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc0f52f0b3b547094f90472a203572fc/A of fc0f52f0b3b547094f90472a203572fc into 0fb5d76cefa24a5dae6fb5ee1cfc38bc(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:07,334 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:07,334 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., storeName=fc0f52f0b3b547094f90472a203572fc/A, priority=13, startTime=1731521646902; duration=0sec 2024-11-13T18:14:07,334 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:07,334 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc0f52f0b3b547094f90472a203572fc:A 2024-11-13T18:14:07,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:07,398 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111302e3cb9c310c4fe484b92eb2e92d01bd_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302e3cb9c310c4fe484b92eb2e92d01bd_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/343e6cee25084d85bb6fdc1f78a057bf, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/343e6cee25084d85bb6fdc1f78a057bf is 175, key is test_row_0/A:col10/1731521645214/Put/seqid=0 2024-11-13T18:14:07,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742343_1519 (size=31255) 2024-11-13T18:14:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:07,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. as already flushing 2024-11-13T18:14:07,416 INFO [Thread-1721 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1721. 2024-11-13T18:14:07,416 DEBUG [Thread-1721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:07,416 DEBUG [Thread-1721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:07,417 INFO [Thread-1719 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1719. 2024-11-13T18:14:07,417 DEBUG [Thread-1721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:07,417 DEBUG [Thread-1719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:07,417 DEBUG [Thread-1719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:07,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:07,417 INFO [Thread-1723 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1723. 2024-11-13T18:14:07,417 DEBUG [Thread-1719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:07,417 DEBUG [Thread-1723 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:07,417 DEBUG [Thread-1723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:07,417 DEBUG [Thread-1723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:07,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:07,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:07,418 INFO [Thread-1727 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1727. 2024-11-13T18:14:07,418 DEBUG [Thread-1727 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:07,418 DEBUG [Thread-1727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:07,418 DEBUG [Thread-1727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:07,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:07,425 INFO [Thread-1725 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1725. 2024-11-13T18:14:07,425 DEBUG [Thread-1725 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:07,425 DEBUG [Thread-1725 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:07,425 DEBUG [Thread-1725 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:07,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:14:07,803 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=466, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/343e6cee25084d85bb6fdc1f78a057bf 2024-11-13T18:14:07,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/3234b3c679414ee8b0dad3bea02412c1 is 50, key is test_row_0/B:col10/1731521645214/Put/seqid=0 2024-11-13T18:14:07,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742344_1520 (size=12301) 2024-11-13T18:14:08,213 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/3234b3c679414ee8b0dad3bea02412c1 2024-11-13T18:14:08,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/b178aa98711048bfbb950cad3743c21b is 50, key is test_row_0/C:col10/1731521645214/Put/seqid=0 2024-11-13T18:14:08,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742345_1521 (size=12301) 2024-11-13T18:14:08,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8c0a274ed76548d0876da72b23af709c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/efd970fd26784d7d81e54900417b3878, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/39f79203dffe49468069a7c2e6b588de, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ef31d3769b4e46169df4a9a33d201d5d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/548ba38d1475431d9b657f9106c44e03, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b969f66f170a4109b2cd09829fe7862b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/5d018941fb824601a5441f4d51f29c39, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/dd7e79910e024817a6de7288702279c4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c027b530a3af4513ba304ce9a2d78df8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b95c72a0082543f389f1abaf43d12e11, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c6c6dec3385c4cadb01aed2fe0aca08a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/7b0bfe58499240ad92e63f4f1dfa45a2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b262be014e9b4738b254912f4482a7d4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/cfd734ce8edb4c06a92656567cf087ed, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/67ea364880a94d8ab3d63212c45ab65b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/658a01b5099a4a6d84e0acdab8092205, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/81ef485750064d68bce4e5b93dbc2a5d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f96e5f710b4b4793b02e258f772feb5c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/086adaa32e224f498461578ec7ffe708, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/a304c8cd8fd841f89d98b1ca2ca5e43c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/1cba0a2aa6294404a1776b617297dc55, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/10166af818d14027b9960148fc5a799c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/e358de78d20c433084c36c328309c1d9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6c4b81df5c1741ff8d0c0ec62768e9ff, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/9d2c7a053ee74e3c935d2ce1f7ba89a5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3d554e31a359477dbb81f67cdaffc5ef, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/03d7ebb3824544f88d00d9f037f39a88, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b847a59ef3ca425589246776108972d4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8a87973c68d14a238f461d6eb0d827eb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3beccb25477f45abbaac4aa1d3061dbe] to archive 2024-11-13T18:14:08,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:14:08,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8c0a274ed76548d0876da72b23af709c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8c0a274ed76548d0876da72b23af709c 2024-11-13T18:14:08,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ddc0e5438cec4c4d97c4c9db9f8e3fd0 2024-11-13T18:14:08,560 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/efd970fd26784d7d81e54900417b3878 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/efd970fd26784d7d81e54900417b3878 2024-11-13T18:14:08,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/39f79203dffe49468069a7c2e6b588de to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/39f79203dffe49468069a7c2e6b588de 2024-11-13T18:14:08,563 WARN [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-13T18:14:08,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ef31d3769b4e46169df4a9a33d201d5d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/ef31d3769b4e46169df4a9a33d201d5d 2024-11-13T18:14:08,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/548ba38d1475431d9b657f9106c44e03 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/548ba38d1475431d9b657f9106c44e03 2024-11-13T18:14:08,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b969f66f170a4109b2cd09829fe7862b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b969f66f170a4109b2cd09829fe7862b 2024-11-13T18:14:08,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/5d018941fb824601a5441f4d51f29c39 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/5d018941fb824601a5441f4d51f29c39 2024-11-13T18:14:08,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/dd7e79910e024817a6de7288702279c4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/dd7e79910e024817a6de7288702279c4 2024-11-13T18:14:08,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c027b530a3af4513ba304ce9a2d78df8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c027b530a3af4513ba304ce9a2d78df8 2024-11-13T18:14:08,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b95c72a0082543f389f1abaf43d12e11 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b95c72a0082543f389f1abaf43d12e11 2024-11-13T18:14:08,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c6c6dec3385c4cadb01aed2fe0aca08a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/c6c6dec3385c4cadb01aed2fe0aca08a 2024-11-13T18:14:08,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/7b0bfe58499240ad92e63f4f1dfa45a2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/7b0bfe58499240ad92e63f4f1dfa45a2 2024-11-13T18:14:08,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6bd0ef00bd814c3ba9a6bc360ae7a4e0 2024-11-13T18:14:08,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b262be014e9b4738b254912f4482a7d4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b262be014e9b4738b254912f4482a7d4 2024-11-13T18:14:08,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/cfd734ce8edb4c06a92656567cf087ed to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/cfd734ce8edb4c06a92656567cf087ed 2024-11-13T18:14:08,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/eb70b1ec1b9d4d83b1a46b7b3bc4c27e 2024-11-13T18:14:08,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/67ea364880a94d8ab3d63212c45ab65b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/67ea364880a94d8ab3d63212c45ab65b 2024-11-13T18:14:08,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/658a01b5099a4a6d84e0acdab8092205 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/658a01b5099a4a6d84e0acdab8092205 2024-11-13T18:14:08,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/81ef485750064d68bce4e5b93dbc2a5d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/81ef485750064d68bce4e5b93dbc2a5d 2024-11-13T18:14:08,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f96e5f710b4b4793b02e258f772feb5c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f96e5f710b4b4793b02e258f772feb5c 2024-11-13T18:14:08,583 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/086adaa32e224f498461578ec7ffe708 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/086adaa32e224f498461578ec7ffe708 2024-11-13T18:14:08,584 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/a304c8cd8fd841f89d98b1ca2ca5e43c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/a304c8cd8fd841f89d98b1ca2ca5e43c 2024-11-13T18:14:08,585 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/1cba0a2aa6294404a1776b617297dc55 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/1cba0a2aa6294404a1776b617297dc55 2024-11-13T18:14:08,586 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/10166af818d14027b9960148fc5a799c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/10166af818d14027b9960148fc5a799c 2024-11-13T18:14:08,587 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/e358de78d20c433084c36c328309c1d9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/e358de78d20c433084c36c328309c1d9 2024-11-13T18:14:08,589 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6c4b81df5c1741ff8d0c0ec62768e9ff to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/6c4b81df5c1741ff8d0c0ec62768e9ff 2024-11-13T18:14:08,590 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/9d2c7a053ee74e3c935d2ce1f7ba89a5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/9d2c7a053ee74e3c935d2ce1f7ba89a5 2024-11-13T18:14:08,591 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3d554e31a359477dbb81f67cdaffc5ef to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3d554e31a359477dbb81f67cdaffc5ef 2024-11-13T18:14:08,592 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/03d7ebb3824544f88d00d9f037f39a88 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/03d7ebb3824544f88d00d9f037f39a88 2024-11-13T18:14:08,593 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b847a59ef3ca425589246776108972d4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/b847a59ef3ca425589246776108972d4 2024-11-13T18:14:08,594 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8a87973c68d14a238f461d6eb0d827eb to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/8a87973c68d14a238f461d6eb0d827eb 2024-11-13T18:14:08,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3beccb25477f45abbaac4aa1d3061dbe to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/3beccb25477f45abbaac4aa1d3061dbe 2024-11-13T18:14:08,599 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/dc9dc39bb00a4503b98f40495f41507a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b44cf2848b2c4c21adb94e15b760f896, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/02539b7ef98242cfb4b931c074c7306c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/388b886260954ae1b8173b5908a8592f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6382c2ebfd7343829b0dc4eafb138677, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a91efec4e8154658a135fe3c226f6912, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/ef0e92354c13470dadaf908ec68626a3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2010fc8e14de4c60977b5c05f9b1f0bf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/d2d50797e64849e0a0d085b075305bc3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/8a127a4c2c5044dc95f49ca74fee57c5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c48995a7746744f58841247a7c76c139, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/497366dec20a4d2f96620cb833f1ff11, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/19c62ced4df44fd1abf02034e2a19acf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6ddd4dd4adad4566894fb3e5b0870777, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/87e1a5953eb04ddc9ca14a2c8e8b8721, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/4f909f109c104e3796b178ef12b8cfb7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b85edce65eff4786b61c123b0330469e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/233ad71fe17c4c25a52ba07d9a9ed476, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2d71e6fe8ab444dc84082798549cf5f3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/486ceced31e7447393637fb3d223d659, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c0bce5c8b51e484cabed36b8588e42ed, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/11dd8e9bad60463eb35db58ce4cd3289, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2f4ad2fe0aad4d7783b3f1a82886f077, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/94b8079ddec0421fa794dd604a24536e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/27cf419bdd4d4fe0bc0a97ade31d9747, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/88206899ef314701917750175fc87a69, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/39b9a29b41c14aafa46e9edfa8aee489, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/fdc59f4ebe8f429fbdafabe2a0643688, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/0d2e2da14e9e45899284bcca1f2af7af, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/e3e8e07d25254940926971d592d5708c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2a13be6b511648a8b470852a744a6197, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/62fa0ac507c046d89aea521c1340032b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a3d4fc0e57ac494685c82f51fc5b0d55] to archive 2024-11-13T18:14:08,600 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:14:08,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/dc9dc39bb00a4503b98f40495f41507a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/dc9dc39bb00a4503b98f40495f41507a 2024-11-13T18:14:08,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b44cf2848b2c4c21adb94e15b760f896 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b44cf2848b2c4c21adb94e15b760f896 2024-11-13T18:14:08,603 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/02539b7ef98242cfb4b931c074c7306c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/02539b7ef98242cfb4b931c074c7306c 2024-11-13T18:14:08,604 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/388b886260954ae1b8173b5908a8592f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/388b886260954ae1b8173b5908a8592f 2024-11-13T18:14:08,606 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6382c2ebfd7343829b0dc4eafb138677 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6382c2ebfd7343829b0dc4eafb138677 2024-11-13T18:14:08,607 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a91efec4e8154658a135fe3c226f6912 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a91efec4e8154658a135fe3c226f6912 2024-11-13T18:14:08,608 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/ef0e92354c13470dadaf908ec68626a3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/ef0e92354c13470dadaf908ec68626a3 2024-11-13T18:14:08,608 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2010fc8e14de4c60977b5c05f9b1f0bf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2010fc8e14de4c60977b5c05f9b1f0bf 2024-11-13T18:14:08,609 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/d2d50797e64849e0a0d085b075305bc3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/d2d50797e64849e0a0d085b075305bc3 2024-11-13T18:14:08,610 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/8a127a4c2c5044dc95f49ca74fee57c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/8a127a4c2c5044dc95f49ca74fee57c5 2024-11-13T18:14:08,611 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c48995a7746744f58841247a7c76c139 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c48995a7746744f58841247a7c76c139 2024-11-13T18:14:08,612 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/497366dec20a4d2f96620cb833f1ff11 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/497366dec20a4d2f96620cb833f1ff11 2024-11-13T18:14:08,613 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/19c62ced4df44fd1abf02034e2a19acf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/19c62ced4df44fd1abf02034e2a19acf 2024-11-13T18:14:08,614 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6ddd4dd4adad4566894fb3e5b0870777 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/6ddd4dd4adad4566894fb3e5b0870777 2024-11-13T18:14:08,615 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/87e1a5953eb04ddc9ca14a2c8e8b8721 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/87e1a5953eb04ddc9ca14a2c8e8b8721 2024-11-13T18:14:08,615 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/4f909f109c104e3796b178ef12b8cfb7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/4f909f109c104e3796b178ef12b8cfb7 2024-11-13T18:14:08,616 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b85edce65eff4786b61c123b0330469e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/b85edce65eff4786b61c123b0330469e 2024-11-13T18:14:08,617 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/233ad71fe17c4c25a52ba07d9a9ed476 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/233ad71fe17c4c25a52ba07d9a9ed476 2024-11-13T18:14:08,618 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2d71e6fe8ab444dc84082798549cf5f3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2d71e6fe8ab444dc84082798549cf5f3 2024-11-13T18:14:08,619 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/486ceced31e7447393637fb3d223d659 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/486ceced31e7447393637fb3d223d659 2024-11-13T18:14:08,620 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c0bce5c8b51e484cabed36b8588e42ed to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/c0bce5c8b51e484cabed36b8588e42ed 2024-11-13T18:14:08,621 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/11dd8e9bad60463eb35db58ce4cd3289 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/11dd8e9bad60463eb35db58ce4cd3289 2024-11-13T18:14:08,622 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2f4ad2fe0aad4d7783b3f1a82886f077 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2f4ad2fe0aad4d7783b3f1a82886f077 2024-11-13T18:14:08,623 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/94b8079ddec0421fa794dd604a24536e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/94b8079ddec0421fa794dd604a24536e 2024-11-13T18:14:08,623 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/b178aa98711048bfbb950cad3743c21b 2024-11-13T18:14:08,624 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/27cf419bdd4d4fe0bc0a97ade31d9747 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/27cf419bdd4d4fe0bc0a97ade31d9747 2024-11-13T18:14:08,625 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/88206899ef314701917750175fc87a69 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/88206899ef314701917750175fc87a69 2024-11-13T18:14:08,626 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/39b9a29b41c14aafa46e9edfa8aee489 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/39b9a29b41c14aafa46e9edfa8aee489 2024-11-13T18:14:08,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/343e6cee25084d85bb6fdc1f78a057bf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/343e6cee25084d85bb6fdc1f78a057bf 2024-11-13T18:14:08,627 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/fdc59f4ebe8f429fbdafabe2a0643688 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/fdc59f4ebe8f429fbdafabe2a0643688 2024-11-13T18:14:08,629 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/0d2e2da14e9e45899284bcca1f2af7af to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/0d2e2da14e9e45899284bcca1f2af7af 2024-11-13T18:14:08,631 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/e3e8e07d25254940926971d592d5708c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/e3e8e07d25254940926971d592d5708c 2024-11-13T18:14:08,631 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/343e6cee25084d85bb6fdc1f78a057bf, entries=150, sequenceid=466, filesize=30.5 K 2024-11-13T18:14:08,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/3234b3c679414ee8b0dad3bea02412c1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/3234b3c679414ee8b0dad3bea02412c1 2024-11-13T18:14:08,632 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2a13be6b511648a8b470852a744a6197 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/2a13be6b511648a8b470852a744a6197 2024-11-13T18:14:08,633 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/62fa0ac507c046d89aea521c1340032b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/62fa0ac507c046d89aea521c1340032b 2024-11-13T18:14:08,635 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a3d4fc0e57ac494685c82f51fc5b0d55 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a3d4fc0e57ac494685c82f51fc5b0d55 2024-11-13T18:14:08,635 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/3234b3c679414ee8b0dad3bea02412c1, entries=150, sequenceid=466, filesize=12.0 K 2024-11-13T18:14:08,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/b178aa98711048bfbb950cad3743c21b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/b178aa98711048bfbb950cad3743c21b 2024-11-13T18:14:08,639 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ed5f6ed4d3a4451486ab8d748444499d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/39dcc7667eef494e8b9e5558c57e7226, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a36f51b1961e435aa51118a4283afecd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74c5b0cf33bf452db8a07b6af43a0cb3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c319efe19b3453e9f4b08473b05997f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84510b9c4bf244eeb1606e60f642c073, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d8106c3fe0f645648042eccc93dc632b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f995ea63ec974c4ebfa964c9d11db64a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/df27bd3d629c4517830cee5099812a18, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/dac98bf7088d48edb3c77c00537eb054, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84e002eec6454e2e9b8b5dc20946a383, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74deeb3966af46d69f3cac808c446774, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/62e8d8bd7c0842078d9e48426158a58a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/32a50d325629475e967f531e9cebfcbf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/07bc165bab6146dd9f7abe3f913e8403, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1d0dd7c1784047128ffd86d275dcb087, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d49bfd40718b4281be92ef57c53702b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e9c21d205bac459ba938e0c4a843bbbf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/390f851ab7cd450f9c9ca724374057aa, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7656ba08010849a9a8c8779dc08d202e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/b14790eb338f49788149ff17c7491b82, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a24e76e90f544ad49ac8eadaaf766b42, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e09556a8b2284717a4db51291b0b2ddb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2985918bbb6e4eed846aa2c49cb4bbea, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2b44ae4555c740d49bebb6453590c18c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f029ff2dd1014424888bb1607ff9a57d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/c9cb765eabdf4549bb6e0755515e26e3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c76cec9bf774448957be46d7bc0a471, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ac0b438cc4da4367b4de80a3cc7f040c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7496cab9e041410dabd7e9ff13ce423c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a54f549a8be14a80bdada488d7fef13e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/0f8a3380909a422689de38983c14f806] to archive 2024-11-13T18:14:08,639 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/b178aa98711048bfbb950cad3743c21b, entries=150, sequenceid=466, filesize=12.0 K 2024-11-13T18:14:08,640 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:14:08,640 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=33.54 KB/34350 for fc0f52f0b3b547094f90472a203572fc in 1680ms, sequenceid=466, compaction requested=false 2024-11-13T18:14:08,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for fc0f52f0b3b547094f90472a203572fc: 2024-11-13T18:14:08,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:08,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-13T18:14:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-13T18:14:08,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ed5f6ed4d3a4451486ab8d748444499d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ed5f6ed4d3a4451486ab8d748444499d 2024-11-13T18:14:08,643 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/39dcc7667eef494e8b9e5558c57e7226 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/39dcc7667eef494e8b9e5558c57e7226 2024-11-13T18:14:08,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-13T18:14:08,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2240 sec 2024-11-13T18:14:08,644 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a36f51b1961e435aa51118a4283afecd to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a36f51b1961e435aa51118a4283afecd 2024-11-13T18:14:08,645 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74c5b0cf33bf452db8a07b6af43a0cb3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74c5b0cf33bf452db8a07b6af43a0cb3 2024-11-13T18:14:08,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.2300 sec 2024-11-13T18:14:08,646 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c319efe19b3453e9f4b08473b05997f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c319efe19b3453e9f4b08473b05997f 2024-11-13T18:14:08,647 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84510b9c4bf244eeb1606e60f642c073 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84510b9c4bf244eeb1606e60f642c073 2024-11-13T18:14:08,648 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d8106c3fe0f645648042eccc93dc632b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d8106c3fe0f645648042eccc93dc632b 2024-11-13T18:14:08,650 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f995ea63ec974c4ebfa964c9d11db64a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f995ea63ec974c4ebfa964c9d11db64a 2024-11-13T18:14:08,651 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/df27bd3d629c4517830cee5099812a18 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/df27bd3d629c4517830cee5099812a18 2024-11-13T18:14:08,651 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/dac98bf7088d48edb3c77c00537eb054 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/dac98bf7088d48edb3c77c00537eb054 2024-11-13T18:14:08,653 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84e002eec6454e2e9b8b5dc20946a383 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/84e002eec6454e2e9b8b5dc20946a383 2024-11-13T18:14:08,654 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74deeb3966af46d69f3cac808c446774 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/74deeb3966af46d69f3cac808c446774 2024-11-13T18:14:08,655 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/62e8d8bd7c0842078d9e48426158a58a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/62e8d8bd7c0842078d9e48426158a58a 2024-11-13T18:14:08,656 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/32a50d325629475e967f531e9cebfcbf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/32a50d325629475e967f531e9cebfcbf 2024-11-13T18:14:08,661 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/07bc165bab6146dd9f7abe3f913e8403 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/07bc165bab6146dd9f7abe3f913e8403 2024-11-13T18:14:08,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1d0dd7c1784047128ffd86d275dcb087 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1d0dd7c1784047128ffd86d275dcb087 2024-11-13T18:14:08,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d49bfd40718b4281be92ef57c53702b5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/d49bfd40718b4281be92ef57c53702b5 2024-11-13T18:14:08,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e9c21d205bac459ba938e0c4a843bbbf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e9c21d205bac459ba938e0c4a843bbbf 2024-11-13T18:14:08,666 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/390f851ab7cd450f9c9ca724374057aa to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/390f851ab7cd450f9c9ca724374057aa 2024-11-13T18:14:08,667 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7656ba08010849a9a8c8779dc08d202e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7656ba08010849a9a8c8779dc08d202e 2024-11-13T18:14:08,670 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/b14790eb338f49788149ff17c7491b82 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/b14790eb338f49788149ff17c7491b82 2024-11-13T18:14:08,671 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a24e76e90f544ad49ac8eadaaf766b42 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a24e76e90f544ad49ac8eadaaf766b42 2024-11-13T18:14:08,672 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e09556a8b2284717a4db51291b0b2ddb to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/e09556a8b2284717a4db51291b0b2ddb 2024-11-13T18:14:08,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2985918bbb6e4eed846aa2c49cb4bbea to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2985918bbb6e4eed846aa2c49cb4bbea 2024-11-13T18:14:08,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2b44ae4555c740d49bebb6453590c18c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/2b44ae4555c740d49bebb6453590c18c 2024-11-13T18:14:08,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f029ff2dd1014424888bb1607ff9a57d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/f029ff2dd1014424888bb1607ff9a57d 2024-11-13T18:14:08,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/c9cb765eabdf4549bb6e0755515e26e3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/c9cb765eabdf4549bb6e0755515e26e3 2024-11-13T18:14:08,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c76cec9bf774448957be46d7bc0a471 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/1c76cec9bf774448957be46d7bc0a471 2024-11-13T18:14:08,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ac0b438cc4da4367b4de80a3cc7f040c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ac0b438cc4da4367b4de80a3cc7f040c 2024-11-13T18:14:08,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7496cab9e041410dabd7e9ff13ce423c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/7496cab9e041410dabd7e9ff13ce423c 2024-11-13T18:14:08,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a54f549a8be14a80bdada488d7fef13e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/a54f549a8be14a80bdada488d7fef13e 2024-11-13T18:14:08,687 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/0f8a3380909a422689de38983c14f806 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/0f8a3380909a422689de38983c14f806 2024-11-13T18:14:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:14:09,575 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 880 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2640 rows 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 872 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2616 rows 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 871 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2613 rows 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 861 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2583 rows 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 873 2024-11-13T18:14:09,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2617 rows 2024-11-13T18:14:09,575 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:14:09,575 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:14:09,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:09,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:09,576 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:09,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:14:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:14:09,580 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521649580"}]},"ts":"1731521649580"} 2024-11-13T18:14:09,582 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:14:09,582 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:14:09,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:14:09,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, UNASSIGN}] 2024-11-13T18:14:09,585 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, UNASSIGN 2024-11-13T18:14:09,586 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=fc0f52f0b3b547094f90472a203572fc, regionState=CLOSING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:09,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, UNASSIGN because future has completed 2024-11-13T18:14:09,588 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:14:09,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:14:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:14:09,741 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing fc0f52f0b3b547094f90472a203572fc, disabling compactions & flushes 2024-11-13T18:14:09,741 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. after waiting 0 ms 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:09,741 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing fc0f52f0b3b547094f90472a203572fc 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=A 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=B 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc0f52f0b3b547094f90472a203572fc, store=C 2024-11-13T18:14:09,741 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:09,747 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111306707e47377c4822ae9cb673b0db50cd_fc0f52f0b3b547094f90472a203572fc is 50, key is test_row_0/A:col10/1731521647415/Put/seqid=0 2024-11-13T18:14:09,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742346_1522 (size=12454) 2024-11-13T18:14:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:14:10,152 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:10,156 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111306707e47377c4822ae9cb673b0db50cd_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111306707e47377c4822ae9cb673b0db50cd_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:10,157 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/f4cfc95e8d6e4e809434f9b66b0c4077, store: [table=TestAcidGuarantees family=A region=fc0f52f0b3b547094f90472a203572fc] 2024-11-13T18:14:10,157 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/f4cfc95e8d6e4e809434f9b66b0c4077 is 175, key is test_row_0/A:col10/1731521647415/Put/seqid=0 2024-11-13T18:14:10,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742347_1523 (size=31255) 2024-11-13T18:14:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:14:10,567 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=475, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/f4cfc95e8d6e4e809434f9b66b0c4077 2024-11-13T18:14:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/0a1050e531284ace84120acf0c75fe76 is 50, key is test_row_0/B:col10/1731521647415/Put/seqid=0 2024-11-13T18:14:10,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742348_1524 (size=12301) 2024-11-13T18:14:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:14:10,977 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/0a1050e531284ace84120acf0c75fe76 2024-11-13T18:14:10,984 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/21f71901673741a6a492e2f2517e4302 is 50, key is test_row_0/C:col10/1731521647415/Put/seqid=0 2024-11-13T18:14:10,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742349_1525 (size=12301) 2024-11-13T18:14:11,387 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/21f71901673741a6a492e2f2517e4302 2024-11-13T18:14:11,392 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/A/f4cfc95e8d6e4e809434f9b66b0c4077 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f4cfc95e8d6e4e809434f9b66b0c4077 2024-11-13T18:14:11,396 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f4cfc95e8d6e4e809434f9b66b0c4077, entries=150, sequenceid=475, filesize=30.5 K 2024-11-13T18:14:11,397 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/B/0a1050e531284ace84120acf0c75fe76 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/0a1050e531284ace84120acf0c75fe76 2024-11-13T18:14:11,400 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/0a1050e531284ace84120acf0c75fe76, entries=150, sequenceid=475, filesize=12.0 K 2024-11-13T18:14:11,400 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/.tmp/C/21f71901673741a6a492e2f2517e4302 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/21f71901673741a6a492e2f2517e4302 2024-11-13T18:14:11,403 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/21f71901673741a6a492e2f2517e4302, entries=150, sequenceid=475, filesize=12.0 K 2024-11-13T18:14:11,404 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for fc0f52f0b3b547094f90472a203572fc in 1663ms, sequenceid=475, compaction requested=true 2024-11-13T18:14:11,409 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/recovered.edits/478.seqid, newMaxSeqId=478, maxSeqId=4 2024-11-13T18:14:11,409 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. 2024-11-13T18:14:11,410 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for fc0f52f0b3b547094f90472a203572fc: Waiting for close lock at 1731521649741Running coprocessor pre-close hooks at 1731521649741Disabling compacts and flushes for region at 1731521649741Disabling writes for close at 1731521649741Obtaining lock to block concurrent updates at 1731521649741Preparing flush snapshotting stores in fc0f52f0b3b547094f90472a203572fc at 1731521649741Finished memstore snapshotting TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731521649741Flushing stores of TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc. at 1731521649742 (+1 ms)Flushing fc0f52f0b3b547094f90472a203572fc/A: creating writer at 1731521649742Flushing mob file fc0f52f0b3b547094f90472a203572fc/A: appending metadata at 1731521649747 (+5 ms)Flushing mob file fc0f52f0b3b547094f90472a203572fc/A: closing flushed file at 1731521649747Flushing fc0f52f0b3b547094f90472a203572fc/A: appending metadata at 1731521650157 (+410 ms)Flushing fc0f52f0b3b547094f90472a203572fc/A: closing flushed file at 1731521650157Flushing fc0f52f0b3b547094f90472a203572fc/B: creating writer at 1731521650571 (+414 ms)Flushing fc0f52f0b3b547094f90472a203572fc/B: appending metadata at 1731521650573 (+2 ms)Flushing fc0f52f0b3b547094f90472a203572fc/B: closing flushed file at 1731521650573Flushing fc0f52f0b3b547094f90472a203572fc/C: creating writer at 1731521650981 (+408 ms)Flushing fc0f52f0b3b547094f90472a203572fc/C: appending metadata at 1731521650983 (+2 ms)Flushing fc0f52f0b3b547094f90472a203572fc/C: closing flushed file at 1731521650983Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3643b1c: reopening flushed file at 1731521651391 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55d7ca56: reopening flushed file at 1731521651396 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57e36cb2: reopening flushed file at 1731521651400 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for fc0f52f0b3b547094f90472a203572fc in 1663ms, sequenceid=475, compaction requested=true at 1731521651404 (+4 ms)Writing region close event to WAL at 1731521651406 (+2 ms)Running coprocessor post-close hooks at 1731521651409 (+3 ms)Closed at 1731521651409 2024-11-13T18:14:11,412 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,413 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=fc0f52f0b3b547094f90472a203572fc, regionState=CLOSED 2024-11-13T18:14:11,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:14:11,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-13T18:14:11,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure fc0f52f0b3b547094f90472a203572fc, server=fda96cf025dc,44483,1731521526892 in 1.8280 sec 2024-11-13T18:14:11,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-13T18:14:11,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc0f52f0b3b547094f90472a203572fc, UNASSIGN in 1.8340 sec 2024-11-13T18:14:11,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-13T18:14:11,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8380 sec 2024-11-13T18:14:11,425 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521651425"}]},"ts":"1731521651425"} 2024-11-13T18:14:11,427 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:14:11,427 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:14:11,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8520 sec 2024-11-13T18:14:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:14:11,725 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:11,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:14:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:11,727 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T18:14:11,728 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:11,729 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,731 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/recovered.edits] 2024-11-13T18:14:11,733 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/0fb5d76cefa24a5dae6fb5ee1cfc38bc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/0fb5d76cefa24a5dae6fb5ee1cfc38bc 2024-11-13T18:14:11,735 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/343e6cee25084d85bb6fdc1f78a057bf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/343e6cee25084d85bb6fdc1f78a057bf 2024-11-13T18:14:11,735 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f4cfc95e8d6e4e809434f9b66b0c4077 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/A/f4cfc95e8d6e4e809434f9b66b0c4077 2024-11-13T18:14:11,737 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/0a1050e531284ace84120acf0c75fe76 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/0a1050e531284ace84120acf0c75fe76 2024-11-13T18:14:11,738 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/3234b3c679414ee8b0dad3bea02412c1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/3234b3c679414ee8b0dad3bea02412c1 2024-11-13T18:14:11,739 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a5ddf4c7298b4a75935fc244e07a5811 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/B/a5ddf4c7298b4a75935fc244e07a5811 2024-11-13T18:14:11,741 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/21f71901673741a6a492e2f2517e4302 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/21f71901673741a6a492e2f2517e4302 2024-11-13T18:14:11,742 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/b178aa98711048bfbb950cad3743c21b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/b178aa98711048bfbb950cad3743c21b 2024-11-13T18:14:11,743 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ccbc2beecd6045a3b18733ae08a98fa0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/C/ccbc2beecd6045a3b18733ae08a98fa0 2024-11-13T18:14:11,745 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/recovered.edits/478.seqid to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc/recovered.edits/478.seqid 2024-11-13T18:14:11,746 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,746 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:14:11,746 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:14:11,747 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T18:14:11,750 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302af861295c14d3cad9e254f916319c3_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302af861295c14d3cad9e254f916319c3_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,751 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302e3cb9c310c4fe484b92eb2e92d01bd_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302e3cb9c310c4fe484b92eb2e92d01bd_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,752 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111306707e47377c4822ae9cb673b0db50cd_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111306707e47377c4822ae9cb673b0db50cd_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,753 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132076668e6ead44aea40c8cbcd6f96d8a_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132076668e6ead44aea40c8cbcd6f96d8a_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,754 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111336ddc027663c4786883c3e47ba92cf63_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111336ddc027663c4786883c3e47ba92cf63_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,755 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133a2c66deb4774cf396b6a5207f4160b5_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133a2c66deb4774cf396b6a5207f4160b5_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,756 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111342a0c15bc352483ea765257fb9c8cde1_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111342a0c15bc352483ea765257fb9c8cde1_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,757 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134a6b1864d3a34afdb22f299d7ad01563_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134a6b1864d3a34afdb22f299d7ad01563_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,759 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113502a62194d804de29a401eb5c00325c7_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113502a62194d804de29a401eb5c00325c7_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,760 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111353dc61ebac7e44b799d7fd491be2a441_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111353dc61ebac7e44b799d7fd491be2a441_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,761 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113635a67e748b640bcb3abc6cddeb496d7_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113635a67e748b640bcb3abc6cddeb496d7_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,762 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136de96acab8eb4f1387b74c33c30f2482_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136de96acab8eb4f1387b74c33c30f2482_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,763 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113709460571a66402a9642312b9847a6cf_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113709460571a66402a9642312b9847a6cf_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,764 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113880de4562a6349e1bc6de9538184874a_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113880de4562a6349e1bc6de9538184874a_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,764 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111391f96b05d0a04796b197f0ed1ebcb2b4_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111391f96b05d0a04796b197f0ed1ebcb2b4_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,765 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111393990f4b746541d0a534e3a760a17a3e_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111393990f4b746541d0a534e3a760a17a3e_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,766 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139bea4de288444661b8339743a7579c47_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139bea4de288444661b8339743a7579c47_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,767 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113af931412539844179107086658be06dc_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113af931412539844179107086658be06dc_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,768 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b619f7621910456398f28912cce2ef0b_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b619f7621910456398f28912cce2ef0b_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,769 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c2d6e94a417e4982ababc5bca58a2672_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c2d6e94a417e4982ababc5bca58a2672_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,770 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c689174a00904411ade188b8f99d9eb7_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c689174a00904411ade188b8f99d9eb7_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,771 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113da0895753285491199379e43a28e901b_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113da0895753285491199379e43a28e901b_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,772 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e799704015da49d493d20085046ddbc7_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e799704015da49d493d20085046ddbc7_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,773 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e8fd4417c2da4f1b94758bf6771a594b_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e8fd4417c2da4f1b94758bf6771a594b_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,774 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f18bc751b7074e6cac4bbd4c5d3ea235_fc0f52f0b3b547094f90472a203572fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f18bc751b7074e6cac4bbd4c5d3ea235_fc0f52f0b3b547094f90472a203572fc 2024-11-13T18:14:11,775 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:14:11,777 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:11,779 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:14:11,781 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:14:11,782 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:11,782 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:14:11,782 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731521651782"}]},"ts":"9223372036854775807"} 2024-11-13T18:14:11,784 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:14:11,784 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fc0f52f0b3b547094f90472a203572fc, NAME => 'TestAcidGuarantees,,1731521624306.fc0f52f0b3b547094f90472a203572fc.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:14:11,784 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:14:11,784 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731521651784"}]},"ts":"9223372036854775807"} 2024-11-13T18:14:11,785 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:14:11,786 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:11,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-11-13T18:14:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T18:14:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:14:11,835 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:11,846 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 246) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1307743371_22 at /127.0.0.1:41018 [Waiting for operation #401] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1307743371_22 at /127.0.0.1:60364 [Waiting for operation #756] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-873439638_22 at /127.0.0.1:41034 [Waiting for operation #386] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-873439638_22 at /127.0.0.1:41020 [Waiting for operation #395] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=778 (was 716) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=899 (was 569) - AvailableMemoryMB LEAK? - 2024-11-13T18:14:11,856 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=778, ProcessCount=11, AvailableMemoryMB=899 2024-11-13T18:14:11,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:14:11,857 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:14:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:11,859 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:14:11,859 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:11,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-13T18:14:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-13T18:14:11,860 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:14:11,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742350_1526 (size=1061) 2024-11-13T18:14:11,866 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1 2024-11-13T18:14:11,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742351_1527 (size=53) 2024-11-13T18:14:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-13T18:14:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-13T18:14:12,271 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:14:12,271 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 278d3a3ebd6be30bccd59b5fe89f892a, disabling compactions & flushes 2024-11-13T18:14:12,271 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:12,271 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:12,271 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. after waiting 0 ms 2024-11-13T18:14:12,271 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:12,271 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:12,271 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 278d3a3ebd6be30bccd59b5fe89f892a: Waiting for close lock at 1731521652271Disabling compacts and flushes for region at 1731521652271Disabling writes for close at 1731521652271Writing region close event to WAL at 1731521652271Closed at 1731521652271 2024-11-13T18:14:12,272 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:14:12,273 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731521652272"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521652272"}]},"ts":"1731521652272"} 2024-11-13T18:14:12,274 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:14:12,275 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:14:12,275 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521652275"}]},"ts":"1731521652275"} 2024-11-13T18:14:12,276 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:14:12,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=278d3a3ebd6be30bccd59b5fe89f892a, ASSIGN}] 2024-11-13T18:14:12,278 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=278d3a3ebd6be30bccd59b5fe89f892a, ASSIGN 2024-11-13T18:14:12,278 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=278d3a3ebd6be30bccd59b5fe89f892a, ASSIGN; state=OFFLINE, location=fda96cf025dc,44483,1731521526892; forceNewPlan=false, retain=false 2024-11-13T18:14:12,429 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=278d3a3ebd6be30bccd59b5fe89f892a, regionState=OPENING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:12,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=278d3a3ebd6be30bccd59b5fe89f892a, ASSIGN because future has completed 2024-11-13T18:14:12,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:14:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-13T18:14:12,587 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:12,587 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:14:12,587 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,587 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:14:12,587 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,587 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,589 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,590 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:14:12,590 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 278d3a3ebd6be30bccd59b5fe89f892a columnFamilyName A 2024-11-13T18:14:12,590 DEBUG [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:12,590 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] regionserver.HStore(327): Store=278d3a3ebd6be30bccd59b5fe89f892a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:14:12,590 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,591 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:14:12,591 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 278d3a3ebd6be30bccd59b5fe89f892a columnFamilyName B 2024-11-13T18:14:12,591 DEBUG [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:12,592 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] regionserver.HStore(327): Store=278d3a3ebd6be30bccd59b5fe89f892a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:14:12,592 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,593 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:14:12,593 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 278d3a3ebd6be30bccd59b5fe89f892a columnFamilyName C 2024-11-13T18:14:12,593 DEBUG [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:12,593 INFO [StoreOpener-278d3a3ebd6be30bccd59b5fe89f892a-1 {}] regionserver.HStore(327): Store=278d3a3ebd6be30bccd59b5fe89f892a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:14:12,593 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:12,593 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,594 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,594 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,595 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,595 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,595 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:14:12,596 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,598 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:14:12,598 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 278d3a3ebd6be30bccd59b5fe89f892a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60512059, jitterRate=-0.09830005466938019}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:14:12,598 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:12,599 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 278d3a3ebd6be30bccd59b5fe89f892a: Running coprocessor pre-open hook at 1731521652588Writing region info on filesystem at 1731521652588Initializing all the Stores at 1731521652588Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521652588Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521652588Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521652588Cleaning up temporary data from old regions at 1731521652595 (+7 ms)Running coprocessor post-open hooks at 1731521652598 (+3 ms)Region opened successfully at 1731521652599 (+1 ms) 2024-11-13T18:14:12,599 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., pid=116, masterSystemTime=1731521652583 2024-11-13T18:14:12,601 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:12,601 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:12,601 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=278d3a3ebd6be30bccd59b5fe89f892a, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:12,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:14:12,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-13T18:14:12,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 in 172 msec 2024-11-13T18:14:12,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-13T18:14:12,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=278d3a3ebd6be30bccd59b5fe89f892a, ASSIGN in 328 msec 2024-11-13T18:14:12,609 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:14:12,609 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521652609"}]},"ts":"1731521652609"} 2024-11-13T18:14:12,610 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:14:12,611 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:14:12,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 754 msec 2024-11-13T18:14:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-13T18:14:12,994 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:12,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c34aee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:12,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:12,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:12,996 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae56015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:12,998 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c415f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:12,999 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,002 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:14:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,37165,1731521525941 2024-11-13T18:14:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6c287d5c 2024-11-13T18:14:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:14:13,003 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:14:13,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b45f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:13,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:13,006 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a5e1d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:13,007 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c402cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c401a20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:13,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:13,009 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363f5beb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:13,010 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a539a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4beee145, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:13,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:13,013 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7afb87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:13,015 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77045767, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b4b57e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:13,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:13,016 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3849acab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:13,018 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7edb95f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@474a7417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:13,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:13,020 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25babd21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:13,021 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77839098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686a02e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:13,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:13,023 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a9e416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:13,024 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dde19cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@773521c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:13,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:13,030 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3688a16b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:13,031 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f791ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5ca2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:13,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:13,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3edb5003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:13,035 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@536aae22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2fc368, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:13,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:13,037 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39610c6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:13,039 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@369ad434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34832895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:13,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:13,041 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7134487c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:13,042 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41907634, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:13,044 DEBUG [Thread-2334 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,044 DEBUG [Thread-2332 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,044 DEBUG [Thread-2336 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,044 DEBUG [Thread-2328 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,045 DEBUG [Thread-2336 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,045 DEBUG [Thread-2336 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:13,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-13T18:14:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:14:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:14:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:14:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:14:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:14:13,050 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:13,051 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,051 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:13,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:13,052 DEBUG [Thread-2347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,052 DEBUG [Thread-2339 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,052 DEBUG [Thread-2339 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,055 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,055 DEBUG [Thread-2347 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,055 DEBUG [Thread-2347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:13,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:14:13,057 DEBUG [Thread-2343 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:13,058 DEBUG [Thread-2345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:14:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:14:13,060 DEBUG [Thread-2345 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,060 DEBUG [Thread-2341 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,060 DEBUG [Thread-2345 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,060 DEBUG [Thread-2341 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,062 DEBUG [Thread-2330 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:14:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:13,065 DEBUG [Thread-2345 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:14:13,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:14:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2] 2024-11-13T18:14:13,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/cdd6af2d5d464293b752fe80554b8bad is 50, key is test_row_0/A:col10/1731521653056/Put/seqid=0 2024-11-13T18:14:13,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731521653081, max timestamp=1731521653102Num uniques 859; 2024-11-13T18:14:13,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T18:14:13,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731521653081, max timestamp=1731521653102Num uniques 859; 2024-11-13T18:14:13,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T18:14:13,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731521653081, max timestamp=1731521653102Num uniques 859; 2024-11-13T18:14:13,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T18:14:13,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742352_1528 (size=12001) 2024-11-13T18:14:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521713138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:14:13,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521713140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521713141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521713142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521713143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521713254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521713264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521713264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521713265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521713265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:14:13,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521713477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521713484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521713484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521713486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521713486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:13,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:13,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/cdd6af2d5d464293b752fe80554b8bad 2024-11-13T18:14:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/47e7ca6a1b4147eb804378bc27166598 is 50, key is test_row_0/B:col10/1731521653056/Put/seqid=0 2024-11-13T18:14:13,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742353_1529 (size=12001) 2024-11-13T18:14:13,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/47e7ca6a1b4147eb804378bc27166598 2024-11-13T18:14:13,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/2784fe404ab0402d9fcbdcfedfde6956 is 50, key is test_row_0/C:col10/1731521653056/Put/seqid=0 2024-11-13T18:14:13,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:13,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:13,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:14:13,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742354_1530 (size=12001) 2024-11-13T18:14:13,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/2784fe404ab0402d9fcbdcfedfde6956 2024-11-13T18:14:13,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/cdd6af2d5d464293b752fe80554b8bad as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/cdd6af2d5d464293b752fe80554b8bad 2024-11-13T18:14:13,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/cdd6af2d5d464293b752fe80554b8bad, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T18:14:13,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/47e7ca6a1b4147eb804378bc27166598 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/47e7ca6a1b4147eb804378bc27166598 2024-11-13T18:14:13,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/47e7ca6a1b4147eb804378bc27166598, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T18:14:13,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/2784fe404ab0402d9fcbdcfedfde6956 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/2784fe404ab0402d9fcbdcfedfde6956 2024-11-13T18:14:13,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/2784fe404ab0402d9fcbdcfedfde6956, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T18:14:13,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 278d3a3ebd6be30bccd59b5fe89f892a in 668ms, sequenceid=12, compaction requested=false 2024-11-13T18:14:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:13,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:13,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-13T18:14:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:13,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/826e22ce6d054215a09b9673ef2cd948 is 50, key is test_row_0/A:col10/1731521653797/Put/seqid=0 2024-11-13T18:14:13,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742355_1531 (size=16681) 2024-11-13T18:14:13,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/826e22ce6d054215a09b9673ef2cd948 2024-11-13T18:14:13,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:13,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:13,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6650f4130ae3486a83a5c4c7cf64b59f is 50, key is test_row_0/B:col10/1731521653797/Put/seqid=0 2024-11-13T18:14:13,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521713819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521713821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521713821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521713821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742356_1532 (size=12001) 2024-11-13T18:14:13,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6650f4130ae3486a83a5c4c7cf64b59f 2024-11-13T18:14:13,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521713840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:13,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/3b4f0dcab9a341909d743e31d4271727 is 50, key is test_row_0/C:col10/1731521653797/Put/seqid=0 2024-11-13T18:14:13,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742357_1533 (size=12001) 2024-11-13T18:14:13,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/3b4f0dcab9a341909d743e31d4271727 2024-11-13T18:14:13,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/826e22ce6d054215a09b9673ef2cd948 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826e22ce6d054215a09b9673ef2cd948 2024-11-13T18:14:13,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826e22ce6d054215a09b9673ef2cd948, entries=250, sequenceid=47, filesize=16.3 K 2024-11-13T18:14:13,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6650f4130ae3486a83a5c4c7cf64b59f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6650f4130ae3486a83a5c4c7cf64b59f 2024-11-13T18:14:13,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6650f4130ae3486a83a5c4c7cf64b59f, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T18:14:13,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/3b4f0dcab9a341909d743e31d4271727 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3b4f0dcab9a341909d743e31d4271727 2024-11-13T18:14:13,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3b4f0dcab9a341909d743e31d4271727, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T18:14:13,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 278d3a3ebd6be30bccd59b5fe89f892a in 111ms, sequenceid=47, compaction requested=false 2024-11-13T18:14:13,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:13,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:14:13,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:13,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:13,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:13,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:13,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:13,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:13,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/dc71c7f9c2a44447973bedb6cf28358a is 50, key is test_row_0/A:col10/1731521653948/Put/seqid=0 2024-11-13T18:14:13,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742358_1534 (size=16681) 2024-11-13T18:14:13,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/dc71c7f9c2a44447973bedb6cf28358a 2024-11-13T18:14:13,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/1db638ee9ca54b37ab3dc9467b37f7c0 is 50, key is test_row_0/B:col10/1731521653948/Put/seqid=0 2024-11-13T18:14:13,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:13,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:13,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:13,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742359_1535 (size=12001) 2024-11-13T18:14:14,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/1db638ee9ca54b37ab3dc9467b37f7c0 2024-11-13T18:14:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521713994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521713996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521713997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521714002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521714003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/c6b09e5e2e034f4fa5bf72d18768749e is 50, key is test_row_0/C:col10/1731521653948/Put/seqid=0 2024-11-13T18:14:14,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742360_1536 (size=12001) 2024-11-13T18:14:14,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/c6b09e5e2e034f4fa5bf72d18768749e 2024-11-13T18:14:14,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/dc71c7f9c2a44447973bedb6cf28358a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/dc71c7f9c2a44447973bedb6cf28358a 2024-11-13T18:14:14,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/dc71c7f9c2a44447973bedb6cf28358a, entries=250, sequenceid=59, filesize=16.3 K 2024-11-13T18:14:14,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/1db638ee9ca54b37ab3dc9467b37f7c0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1db638ee9ca54b37ab3dc9467b37f7c0 2024-11-13T18:14:14,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1db638ee9ca54b37ab3dc9467b37f7c0, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T18:14:14,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/c6b09e5e2e034f4fa5bf72d18768749e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c6b09e5e2e034f4fa5bf72d18768749e 2024-11-13T18:14:14,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c6b09e5e2e034f4fa5bf72d18768749e, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T18:14:14,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 278d3a3ebd6be30bccd59b5fe89f892a in 130ms, sequenceid=59, compaction requested=true 2024-11-13T18:14:14,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:14,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:14,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:14,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:14,080 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:14,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:14,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:14,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:14,081 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:14,082 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:14,082 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:14,082 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:14,082 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:14,082 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,082 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,082 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/cdd6af2d5d464293b752fe80554b8bad, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826e22ce6d054215a09b9673ef2cd948, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/dc71c7f9c2a44447973bedb6cf28358a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=44.3 K 2024-11-13T18:14:14,082 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/47e7ca6a1b4147eb804378bc27166598, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6650f4130ae3486a83a5c4c7cf64b59f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1db638ee9ca54b37ab3dc9467b37f7c0] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=35.2 K 2024-11-13T18:14:14,083 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdd6af2d5d464293b752fe80554b8bad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731521653054 2024-11-13T18:14:14,083 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 47e7ca6a1b4147eb804378bc27166598, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731521653054 2024-11-13T18:14:14,083 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 826e22ce6d054215a09b9673ef2cd948, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731521653137 2024-11-13T18:14:14,083 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6650f4130ae3486a83a5c4c7cf64b59f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731521653142 2024-11-13T18:14:14,083 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc71c7f9c2a44447973bedb6cf28358a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521653818 2024-11-13T18:14:14,083 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1db638ee9ca54b37ab3dc9467b37f7c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521653835 2024-11-13T18:14:14,096 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:14,097 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7cae99703da0433d9fd4c3f36a960a57 is 50, key is test_row_0/A:col10/1731521653948/Put/seqid=0 2024-11-13T18:14:14,111 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:14,111 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/9b40816f72b2411bb2c1f3d144b1ab66 is 50, key is test_row_0/B:col10/1731521653948/Put/seqid=0 2024-11-13T18:14:14,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521653996, max timestamp=1731521654133Num uniques 859; 2024-11-13T18:14:14,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521653996, max timestamp=1731521654133Num uniques 859; 2024-11-13T18:14:14,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:14,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:14,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-13T18:14:14,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:14,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:14,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:14,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:14,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:14,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:14,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:14,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:14,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742361_1537 (size=12104) 2024-11-13T18:14:14,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742362_1538 (size=12104) 2024-11-13T18:14:14,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/e5d511b684894f469959918a7f29f72f is 50, key is test_row_0/A:col10/1731521654134/Put/seqid=0 2024-11-13T18:14:14,166 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7cae99703da0433d9fd4c3f36a960a57 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7cae99703da0433d9fd4c3f36a960a57 2024-11-13T18:14:14,171 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/9b40816f72b2411bb2c1f3d144b1ab66 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/9b40816f72b2411bb2c1f3d144b1ab66 2024-11-13T18:14:14,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742363_1539 (size=12001) 2024-11-13T18:14:14,173 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into 7cae99703da0433d9fd4c3f36a960a57(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:14,173 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:14,173 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=13, startTime=1731521654080; duration=0sec 2024-11-13T18:14:14,173 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:14,173 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:14,174 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:14,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/e5d511b684894f469959918a7f29f72f 2024-11-13T18:14:14,175 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:14,176 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:14,176 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into 9b40816f72b2411bb2c1f3d144b1ab66(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:14,176 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:14,176 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,176 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=13, startTime=1731521654080; duration=0sec 2024-11-13T18:14:14,176 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:14,176 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:14,176 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/2784fe404ab0402d9fcbdcfedfde6956, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3b4f0dcab9a341909d743e31d4271727, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c6b09e5e2e034f4fa5bf72d18768749e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=35.2 K 2024-11-13T18:14:14,177 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2784fe404ab0402d9fcbdcfedfde6956, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731521653054 2024-11-13T18:14:14,181 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b4f0dcab9a341909d743e31d4271727, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731521653142 2024-11-13T18:14:14,182 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6b09e5e2e034f4fa5bf72d18768749e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521653835 2024-11-13T18:14:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:14:14,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/291da3d0a3ff41438e0327117c1f8206 is 50, key is test_row_0/B:col10/1731521654134/Put/seqid=0 2024-11-13T18:14:14,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521714181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521714184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521714190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521714191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,211 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:14,211 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/38af54a893c448f5bf7fc575a5027155 is 50, key is test_row_0/C:col10/1731521653948/Put/seqid=0 2024-11-13T18:14:14,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521714199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742365_1541 (size=12104) 2024-11-13T18:14:14,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742364_1540 (size=12001) 2024-11-13T18:14:14,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:14,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:14,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521714304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521714304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521714321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521714321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521714325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:14,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:14,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521714514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521714514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521714534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521714541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521714545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:14,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:14,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:14,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/291da3d0a3ff41438e0327117c1f8206 2024-11-13T18:14:14,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/c3ee815de299451c9770c5f3a65b7f16 is 50, key is test_row_0/C:col10/1731521654134/Put/seqid=0 2024-11-13T18:14:14,661 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/38af54a893c448f5bf7fc575a5027155 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/38af54a893c448f5bf7fc575a5027155 2024-11-13T18:14:14,673 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into 38af54a893c448f5bf7fc575a5027155(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:14,673 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:14,673 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=13, startTime=1731521654080; duration=0sec 2024-11-13T18:14:14,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742366_1542 (size=12001) 2024-11-13T18:14:14,676 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:14,676 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:14,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/c3ee815de299451c9770c5f3a65b7f16 2024-11-13T18:14:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/e5d511b684894f469959918a7f29f72f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e5d511b684894f469959918a7f29f72f 2024-11-13T18:14:14,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e5d511b684894f469959918a7f29f72f, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T18:14:14,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/291da3d0a3ff41438e0327117c1f8206 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/291da3d0a3ff41438e0327117c1f8206 2024-11-13T18:14:14,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/291da3d0a3ff41438e0327117c1f8206, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T18:14:14,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/c3ee815de299451c9770c5f3a65b7f16 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c3ee815de299451c9770c5f3a65b7f16 2024-11-13T18:14:14,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c3ee815de299451c9770c5f3a65b7f16, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T18:14:14,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 278d3a3ebd6be30bccd59b5fe89f892a in 589ms, sequenceid=81, compaction requested=false 2024-11-13T18:14:14,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:14,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:14:14,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:14,766 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:14:14,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:14,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:14,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:14,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:14,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:14,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:14,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/ddb4da72b2ba4fd896a822819cf09f16 is 50, key is test_row_0/A:col10/1731521654189/Put/seqid=0 2024-11-13T18:14:14,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742367_1543 (size=12001) 2024-11-13T18:14:14,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:14,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521714883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521714880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521714884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521714899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521714899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:14,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:14,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521715004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521715004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521715004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521715014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521715014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:14:15,205 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/ddb4da72b2ba4fd896a822819cf09f16 2024-11-13T18:14:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/1203874cf80b464d807122dc7df3788c is 50, key is test_row_0/B:col10/1731521654189/Put/seqid=0 2024-11-13T18:14:15,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742368_1544 (size=12001) 2024-11-13T18:14:15,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521715224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521715224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521715224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521715224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521715234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521715544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521715544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521715545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521715545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521715555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:15,628 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/1203874cf80b464d807122dc7df3788c 2024-11-13T18:14:15,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/41440fd9fafc488e8b6e7a92b3012429 is 50, key is test_row_0/C:col10/1731521654189/Put/seqid=0 2024-11-13T18:14:15,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742369_1545 (size=12001) 2024-11-13T18:14:15,642 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/41440fd9fafc488e8b6e7a92b3012429 2024-11-13T18:14:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/ddb4da72b2ba4fd896a822819cf09f16 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ddb4da72b2ba4fd896a822819cf09f16 2024-11-13T18:14:15,651 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ddb4da72b2ba4fd896a822819cf09f16, entries=150, sequenceid=104, filesize=11.7 K 2024-11-13T18:14:15,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/1203874cf80b464d807122dc7df3788c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1203874cf80b464d807122dc7df3788c 2024-11-13T18:14:15,656 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1203874cf80b464d807122dc7df3788c, entries=150, sequenceid=104, filesize=11.7 K 2024-11-13T18:14:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/41440fd9fafc488e8b6e7a92b3012429 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41440fd9fafc488e8b6e7a92b3012429 2024-11-13T18:14:15,663 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41440fd9fafc488e8b6e7a92b3012429, entries=150, sequenceid=104, filesize=11.7 K 2024-11-13T18:14:15,664 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 278d3a3ebd6be30bccd59b5fe89f892a in 898ms, sequenceid=104, compaction requested=true 2024-11-13T18:14:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:15,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-13T18:14:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-13T18:14:15,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-13T18:14:15,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6150 sec 2024-11-13T18:14:15,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.6220 sec 2024-11-13T18:14:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:16,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T18:14:16,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:16,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:16,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:16,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:16,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:16,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:16,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/f8dd39c0043743788cfbb7b7410c2510 is 50, key is test_row_0/A:col10/1731521654878/Put/seqid=0 2024-11-13T18:14:16,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742370_1546 (size=14341) 2024-11-13T18:14:16,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/f8dd39c0043743788cfbb7b7410c2510 2024-11-13T18:14:16,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/e8f788ded6224715980e93fa5a133a10 is 50, key is test_row_0/B:col10/1731521654878/Put/seqid=0 2024-11-13T18:14:16,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742371_1547 (size=12001) 2024-11-13T18:14:16,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/e8f788ded6224715980e93fa5a133a10 2024-11-13T18:14:16,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/e497b9f8714d4cb588aea88016f6860a is 50, key is test_row_0/C:col10/1731521654878/Put/seqid=0 2024-11-13T18:14:16,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742372_1548 (size=12001) 2024-11-13T18:14:16,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521716108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521716109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521716116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521716118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521716119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521716224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521716234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521716245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521716245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521716245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521716444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521716454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521716464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521716464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521716464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/e497b9f8714d4cb588aea88016f6860a 2024-11-13T18:14:16,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/f8dd39c0043743788cfbb7b7410c2510 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f8dd39c0043743788cfbb7b7410c2510 2024-11-13T18:14:16,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f8dd39c0043743788cfbb7b7410c2510, entries=200, sequenceid=118, filesize=14.0 K 2024-11-13T18:14:16,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/e8f788ded6224715980e93fa5a133a10 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e8f788ded6224715980e93fa5a133a10 2024-11-13T18:14:16,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e8f788ded6224715980e93fa5a133a10, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T18:14:16,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/e497b9f8714d4cb588aea88016f6860a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/e497b9f8714d4cb588aea88016f6860a 2024-11-13T18:14:16,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/e497b9f8714d4cb588aea88016f6860a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T18:14:16,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 278d3a3ebd6be30bccd59b5fe89f892a in 537ms, sequenceid=118, compaction requested=true 2024-11-13T18:14:16,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:16,608 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:16,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:16,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:16,608 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:16,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:16,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:16,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:16,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:16,621 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:16,621 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:16,621 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:16,621 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7cae99703da0433d9fd4c3f36a960a57, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e5d511b684894f469959918a7f29f72f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ddb4da72b2ba4fd896a822819cf09f16, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f8dd39c0043743788cfbb7b7410c2510] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=49.3 K 2024-11-13T18:14:16,624 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cae99703da0433d9fd4c3f36a960a57, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521653835 2024-11-13T18:14:16,631 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:16,631 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5d511b684894f469959918a7f29f72f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731521653998 2024-11-13T18:14:16,631 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:16,631 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:16,631 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/9b40816f72b2411bb2c1f3d144b1ab66, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/291da3d0a3ff41438e0327117c1f8206, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1203874cf80b464d807122dc7df3788c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e8f788ded6224715980e93fa5a133a10] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=47.0 K 2024-11-13T18:14:16,634 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b40816f72b2411bb2c1f3d144b1ab66, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521653835 2024-11-13T18:14:16,634 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 291da3d0a3ff41438e0327117c1f8206, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731521653998 2024-11-13T18:14:16,634 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddb4da72b2ba4fd896a822819cf09f16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731521654187 2024-11-13T18:14:16,634 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1203874cf80b464d807122dc7df3788c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731521654187 2024-11-13T18:14:16,634 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8dd39c0043743788cfbb7b7410c2510, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731521654878 2024-11-13T18:14:16,637 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f788ded6224715980e93fa5a133a10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731521654878 2024-11-13T18:14:16,664 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:16,664 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/8765ccdc6dd140c2b78722d57da18f55 is 50, key is test_row_0/B:col10/1731521654878/Put/seqid=0 2024-11-13T18:14:16,675 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:16,676 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/36084d7c87cc436796c4c6aea712b79f is 50, key is test_row_0/A:col10/1731521654878/Put/seqid=0 2024-11-13T18:14:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742373_1549 (size=12241) 2024-11-13T18:14:16,685 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/8765ccdc6dd140c2b78722d57da18f55 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/8765ccdc6dd140c2b78722d57da18f55 2024-11-13T18:14:16,697 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into 8765ccdc6dd140c2b78722d57da18f55(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:16,697 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:16,697 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=12, startTime=1731521656608; duration=0sec 2024-11-13T18:14:16,698 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:16,698 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:16,698 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:16,699 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:16,699 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:16,699 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:16,699 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/38af54a893c448f5bf7fc575a5027155, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c3ee815de299451c9770c5f3a65b7f16, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41440fd9fafc488e8b6e7a92b3012429, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/e497b9f8714d4cb588aea88016f6860a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=47.0 K 2024-11-13T18:14:16,700 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 38af54a893c448f5bf7fc575a5027155, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731521653835 2024-11-13T18:14:16,701 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c3ee815de299451c9770c5f3a65b7f16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731521653998 2024-11-13T18:14:16,701 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 41440fd9fafc488e8b6e7a92b3012429, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731521654187 2024-11-13T18:14:16,701 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e497b9f8714d4cb588aea88016f6860a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731521654878 2024-11-13T18:14:16,720 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:16,721 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/9bda1451005a4609921d0122a0484d4f is 50, key is test_row_0/C:col10/1731521654878/Put/seqid=0 2024-11-13T18:14:16,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742374_1550 (size=12241) 2024-11-13T18:14:16,730 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/36084d7c87cc436796c4c6aea712b79f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/36084d7c87cc436796c4c6aea712b79f 2024-11-13T18:14:16,735 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into 36084d7c87cc436796c4c6aea712b79f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:16,735 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:16,735 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=12, startTime=1731521656608; duration=0sec 2024-11-13T18:14:16,735 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:16,735 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:16,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742375_1551 (size=12241) 2024-11-13T18:14:16,755 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/9bda1451005a4609921d0122a0484d4f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/9bda1451005a4609921d0122a0484d4f 2024-11-13T18:14:16,763 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into 9bda1451005a4609921d0122a0484d4f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:16,763 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:16,763 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=12, startTime=1731521656608; duration=0sec 2024-11-13T18:14:16,763 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:16,763 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:16,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:14:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:16,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/1980099dbcc04ee18a54fb3414d24361 is 50, key is test_row_0/A:col10/1731521656770/Put/seqid=0 2024-11-13T18:14:16,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742376_1552 (size=12151) 2024-11-13T18:14:16,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/1980099dbcc04ee18a54fb3414d24361 2024-11-13T18:14:16,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521716820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/806a9b3ab5c0411984dd5bbb927dd923 is 50, key is test_row_0/B:col10/1731521656770/Put/seqid=0 2024-11-13T18:14:16,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521716835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521716836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521716838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742377_1553 (size=12151) 2024-11-13T18:14:16,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/806a9b3ab5c0411984dd5bbb927dd923 2024-11-13T18:14:16,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521716841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:16,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b37a7b37b6844770ad0601021ae0afb1 is 50, key is test_row_0/C:col10/1731521656770/Put/seqid=0 2024-11-13T18:14:16,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742378_1554 (size=12151) 2024-11-13T18:14:16,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b37a7b37b6844770ad0601021ae0afb1 2024-11-13T18:14:16,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/1980099dbcc04ee18a54fb3414d24361 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1980099dbcc04ee18a54fb3414d24361 2024-11-13T18:14:16,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1980099dbcc04ee18a54fb3414d24361, entries=150, sequenceid=140, filesize=11.9 K 2024-11-13T18:14:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/806a9b3ab5c0411984dd5bbb927dd923 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/806a9b3ab5c0411984dd5bbb927dd923 2024-11-13T18:14:16,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/806a9b3ab5c0411984dd5bbb927dd923, entries=150, sequenceid=140, filesize=11.9 K 2024-11-13T18:14:16,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b37a7b37b6844770ad0601021ae0afb1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b37a7b37b6844770ad0601021ae0afb1 2024-11-13T18:14:16,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b37a7b37b6844770ad0601021ae0afb1, entries=150, sequenceid=140, filesize=11.9 K 2024-11-13T18:14:16,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 278d3a3ebd6be30bccd59b5fe89f892a in 141ms, sequenceid=140, compaction requested=false 2024-11-13T18:14:16,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:16,938 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:14:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:16,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:14:16,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:16,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:16,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:16,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:16,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:16,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:16,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/6c629a8424354c199d4747d304e1e8c1 is 50, key is test_row_0/A:col10/1731521656834/Put/seqid=0 2024-11-13T18:14:16,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742379_1555 (size=14541) 2024-11-13T18:14:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521717007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521717009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521717011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521717013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521717013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521717134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521717135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521717135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521717135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521717144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:14:17,214 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:17,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-13T18:14:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:14:17,217 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:17,226 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:17,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:14:17,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521717354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521717354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521717355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521717355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521717355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:14:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/6c629a8424354c199d4747d304e1e8c1 2024-11-13T18:14:17,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/ac81fabb632a474db18bc8add5286440 is 50, key is test_row_0/B:col10/1731521656834/Put/seqid=0 2024-11-13T18:14:17,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742380_1556 (size=12151) 2024-11-13T18:14:17,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/ac81fabb632a474db18bc8add5286440 2024-11-13T18:14:17,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/62f142918fb8417db083d2758216ad13 is 50, key is test_row_0/C:col10/1731521656834/Put/seqid=0 2024-11-13T18:14:17,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742381_1557 (size=12151) 2024-11-13T18:14:17,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/62f142918fb8417db083d2758216ad13 2024-11-13T18:14:17,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/6c629a8424354c199d4747d304e1e8c1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6c629a8424354c199d4747d304e1e8c1 2024-11-13T18:14:17,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6c629a8424354c199d4747d304e1e8c1, entries=200, sequenceid=155, filesize=14.2 K 2024-11-13T18:14:17,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/ac81fabb632a474db18bc8add5286440 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ac81fabb632a474db18bc8add5286440 2024-11-13T18:14:17,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ac81fabb632a474db18bc8add5286440, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T18:14:17,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/62f142918fb8417db083d2758216ad13 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/62f142918fb8417db083d2758216ad13 2024-11-13T18:14:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:14:17,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:14:17,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:17,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/62f142918fb8417db083d2758216ad13, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T18:14:17,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 278d3a3ebd6be30bccd59b5fe89f892a in 605ms, sequenceid=155, compaction requested=true 2024-11-13T18:14:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:17,565 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:17,565 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:17,569 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:17,569 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:17,569 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,569 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/36084d7c87cc436796c4c6aea712b79f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1980099dbcc04ee18a54fb3414d24361, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6c629a8424354c199d4747d304e1e8c1] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=38.0 K 2024-11-13T18:14:17,576 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:17,576 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:17,577 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,577 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/8765ccdc6dd140c2b78722d57da18f55, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/806a9b3ab5c0411984dd5bbb927dd923, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ac81fabb632a474db18bc8add5286440] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=35.7 K 2024-11-13T18:14:17,577 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 36084d7c87cc436796c4c6aea712b79f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731521654878 2024-11-13T18:14:17,577 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8765ccdc6dd140c2b78722d57da18f55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731521654878 2024-11-13T18:14:17,581 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 806a9b3ab5c0411984dd5bbb927dd923, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731521656095 2024-11-13T18:14:17,581 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1980099dbcc04ee18a54fb3414d24361, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731521656095 2024-11-13T18:14:17,584 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac81fabb632a474db18bc8add5286440, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731521656834 2024-11-13T18:14:17,585 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c629a8424354c199d4747d304e1e8c1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731521656807 2024-11-13T18:14:17,615 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:17,615 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:17,616 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/38b40421eee24f549e66e49393cbaf75 is 50, key is test_row_0/B:col10/1731521656834/Put/seqid=0 2024-11-13T18:14:17,616 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/37424b7b647f4059bc824bb45e211eff is 50, key is test_row_0/A:col10/1731521656834/Put/seqid=0 2024-11-13T18:14:17,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742382_1558 (size=12493) 2024-11-13T18:14:17,659 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/38b40421eee24f549e66e49393cbaf75 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/38b40421eee24f549e66e49393cbaf75 2024-11-13T18:14:17,667 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into 38b40421eee24f549e66e49393cbaf75(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:17,667 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:17,667 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=13, startTime=1731521657565; duration=0sec 2024-11-13T18:14:17,667 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:17,667 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:17,667 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:17,669 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:17,670 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:17,670 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,670 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/9bda1451005a4609921d0122a0484d4f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b37a7b37b6844770ad0601021ae0afb1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/62f142918fb8417db083d2758216ad13] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=35.7 K 2024-11-13T18:14:17,670 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bda1451005a4609921d0122a0484d4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731521654878 2024-11-13T18:14:17,671 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b37a7b37b6844770ad0601021ae0afb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731521656095 2024-11-13T18:14:17,671 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62f142918fb8417db083d2758216ad13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731521656834 2024-11-13T18:14:17,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742383_1559 (size=12493) 2024-11-13T18:14:17,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:14:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:17,687 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:17,687 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/ed40bca2e08d426784fdab58cfbb5e33 is 50, key is test_row_0/C:col10/1731521656834/Put/seqid=0 2024-11-13T18:14:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:17,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:14:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742384_1560 (size=12493) 2024-11-13T18:14:17,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/2151ceb4b3334c568db2660ffc502b69 is 50, key is test_row_0/A:col10/1731521657010/Put/seqid=0 2024-11-13T18:14:17,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521717709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521717710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521717714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521717715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521717716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742385_1561 (size=12151) 2024-11-13T18:14:17,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521717824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:14:17,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521717834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521717834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:14:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:17,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:17,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521717844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521717844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:14:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521718045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521718054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521718057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:18,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521718064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:18,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521718069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,106 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/37424b7b647f4059bc824bb45e211eff as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/37424b7b647f4059bc824bb45e211eff 2024-11-13T18:14:18,113 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into 37424b7b647f4059bc824bb45e211eff(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:18,113 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:18,113 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=13, startTime=1731521657565; duration=0sec 2024-11-13T18:14:18,113 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:18,113 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:18,117 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/ed40bca2e08d426784fdab58cfbb5e33 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ed40bca2e08d426784fdab58cfbb5e33 2024-11-13T18:14:18,125 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into ed40bca2e08d426784fdab58cfbb5e33(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:18,125 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:18,125 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=13, startTime=1731521657565; duration=0sec 2024-11-13T18:14:18,125 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:18,125 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:18,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/2151ceb4b3334c568db2660ffc502b69 2024-11-13T18:14:18,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/24b2da40bfc94e419f03d5ec9888d7fc is 50, key is test_row_0/B:col10/1731521657010/Put/seqid=0 2024-11-13T18:14:18,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:14:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742386_1562 (size=12151) 2024-11-13T18:14:18,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:14:18,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:18,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:14:18,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521718369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:18,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521718369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521718384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521718385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:18,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521718385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:18,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:18,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:14:18,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:18,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/24b2da40bfc94e419f03d5ec9888d7fc 2024-11-13T18:14:18,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/13204f8d84e34fd3b366b6b604a4e6cf is 50, key is test_row_0/C:col10/1731521657010/Put/seqid=0 2024-11-13T18:14:18,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742387_1563 (size=12151) 2024-11-13T18:14:18,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/13204f8d84e34fd3b366b6b604a4e6cf 2024-11-13T18:14:18,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/2151ceb4b3334c568db2660ffc502b69 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/2151ceb4b3334c568db2660ffc502b69 2024-11-13T18:14:18,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:14:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/2151ceb4b3334c568db2660ffc502b69, entries=150, sequenceid=176, filesize=11.9 K 2024-11-13T18:14:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:18,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/24b2da40bfc94e419f03d5ec9888d7fc as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/24b2da40bfc94e419f03d5ec9888d7fc 2024-11-13T18:14:18,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/24b2da40bfc94e419f03d5ec9888d7fc, entries=150, sequenceid=176, filesize=11.9 K 2024-11-13T18:14:18,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/13204f8d84e34fd3b366b6b604a4e6cf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/13204f8d84e34fd3b366b6b604a4e6cf 2024-11-13T18:14:18,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/13204f8d84e34fd3b366b6b604a4e6cf, entries=150, sequenceid=176, filesize=11.9 K 2024-11-13T18:14:18,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 278d3a3ebd6be30bccd59b5fe89f892a in 957ms, sequenceid=176, compaction requested=false 2024-11-13T18:14:18,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:18,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:18,786 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:18,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7e5b2bdd6f724bdb8898b4215eb540b1 is 50, key is test_row_0/A:col10/1731521657713/Put/seqid=0 2024-11-13T18:14:18,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742388_1564 (size=12151) 2024-11-13T18:14:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:18,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:18,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521658934, max timestamp=1731521658957Num uniques 859; 2024-11-13T18:14:18,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:14:18,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521658934, max timestamp=1731521658957Num uniques 859; 2024-11-13T18:14:18,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731521658934, max timestamp=1731521658957Num uniques 859; 2024-11-13T18:14:18,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:14:18,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:14:19,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521718989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521718993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521718999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521719001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521719008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521719115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521719124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521719124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521719135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521719135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,203 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7e5b2bdd6f724bdb8898b4215eb540b1 2024-11-13T18:14:19,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/02928dabe32d446a8bcc69c71291cdf1 is 50, key is test_row_0/B:col10/1731521657713/Put/seqid=0 2024-11-13T18:14:19,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742389_1565 (size=12151) 2024-11-13T18:14:19,215 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/02928dabe32d446a8bcc69c71291cdf1 2024-11-13T18:14:19,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/553edf81b563480fbdf25a9a455e6c75 is 50, key is test_row_0/C:col10/1731521657713/Put/seqid=0 2024-11-13T18:14:19,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742390_1566 (size=12151) 2024-11-13T18:14:19,225 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/553edf81b563480fbdf25a9a455e6c75 2024-11-13T18:14:19,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7e5b2bdd6f724bdb8898b4215eb540b1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7e5b2bdd6f724bdb8898b4215eb540b1 2024-11-13T18:14:19,235 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7e5b2bdd6f724bdb8898b4215eb540b1, entries=150, sequenceid=190, filesize=11.9 K 2024-11-13T18:14:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/02928dabe32d446a8bcc69c71291cdf1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/02928dabe32d446a8bcc69c71291cdf1 2024-11-13T18:14:19,242 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/02928dabe32d446a8bcc69c71291cdf1, entries=150, sequenceid=190, filesize=11.9 K 2024-11-13T18:14:19,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/553edf81b563480fbdf25a9a455e6c75 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/553edf81b563480fbdf25a9a455e6c75 2024-11-13T18:14:19,246 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/553edf81b563480fbdf25a9a455e6c75, entries=150, sequenceid=190, filesize=11.9 K 2024-11-13T18:14:19,247 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 278d3a3ebd6be30bccd59b5fe89f892a in 461ms, sequenceid=190, compaction requested=true 2024-11-13T18:14:19,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:19,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:19,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-13T18:14:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-13T18:14:19,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-13T18:14:19,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0230 sec 2024-11-13T18:14:19,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.0400 sec 2024-11-13T18:14:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:19,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-13T18:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:19,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/d3236b745d2b40f8a0bafb53e9c091a0 is 50, key is test_row_0/A:col10/1731521658975/Put/seqid=0 2024-11-13T18:14:19,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742391_1567 (size=14541) 2024-11-13T18:14:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:14:19,364 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:19,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-13T18:14:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:14:19,368 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:19,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:19,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521719367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521719367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521719370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521719386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521719389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:14:19,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521719504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521719504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521719504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521719505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521719505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T18:14:19,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:19,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:19,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:19,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T18:14:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:19,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:14:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521719715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521719715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521719715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521719715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521719717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:19,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:19,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/d3236b745d2b40f8a0bafb53e9c091a0 2024-11-13T18:14:19,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/7caa5894c3bf42f4b85c1a8ff6870499 is 50, key is test_row_0/B:col10/1731521658975/Put/seqid=0 2024-11-13T18:14:19,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742392_1568 (size=12151) 2024-11-13T18:14:19,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/7caa5894c3bf42f4b85c1a8ff6870499 2024-11-13T18:14:19,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/41e99a5c7c6e4ad2a15f2258e6378f72 is 50, key is test_row_0/C:col10/1731521658975/Put/seqid=0 2024-11-13T18:14:19,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742393_1569 (size=12151) 2024-11-13T18:14:19,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T18:14:19,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:19,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:19,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:19,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T18:14:19,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:19,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:19,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:19,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:14:20,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521720024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521720034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521720034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521720034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521720035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T18:14:20,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:20,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:20,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:20,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:20,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:20,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/41e99a5c7c6e4ad2a15f2258e6378f72 2024-11-13T18:14:20,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/d3236b745d2b40f8a0bafb53e9c091a0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d3236b745d2b40f8a0bafb53e9c091a0 2024-11-13T18:14:20,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d3236b745d2b40f8a0bafb53e9c091a0, entries=200, sequenceid=224, filesize=14.2 K 2024-11-13T18:14:20,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/7caa5894c3bf42f4b85c1a8ff6870499 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7caa5894c3bf42f4b85c1a8ff6870499 2024-11-13T18:14:20,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7caa5894c3bf42f4b85c1a8ff6870499, entries=150, sequenceid=224, filesize=11.9 K 2024-11-13T18:14:20,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/41e99a5c7c6e4ad2a15f2258e6378f72 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41e99a5c7c6e4ad2a15f2258e6378f72 2024-11-13T18:14:20,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41e99a5c7c6e4ad2a15f2258e6378f72, entries=150, sequenceid=224, filesize=11.9 K 2024-11-13T18:14:20,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 278d3a3ebd6be30bccd59b5fe89f892a in 935ms, sequenceid=224, compaction requested=true 2024-11-13T18:14:20,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:20,269 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:20,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:20,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:20,269 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:20,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:20,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:20,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:20,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:20,270 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:20,271 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:20,271 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:20,271 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/37424b7b647f4059bc824bb45e211eff, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/2151ceb4b3334c568db2660ffc502b69, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7e5b2bdd6f724bdb8898b4215eb540b1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d3236b745d2b40f8a0bafb53e9c091a0] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=50.1 K 2024-11-13T18:14:20,271 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:20,271 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:20,271 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:20,271 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/38b40421eee24f549e66e49393cbaf75, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/24b2da40bfc94e419f03d5ec9888d7fc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/02928dabe32d446a8bcc69c71291cdf1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7caa5894c3bf42f4b85c1a8ff6870499] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=47.8 K 2024-11-13T18:14:20,272 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37424b7b647f4059bc824bb45e211eff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731521656834 2024-11-13T18:14:20,272 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b40421eee24f549e66e49393cbaf75, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731521656834 2024-11-13T18:14:20,272 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2151ceb4b3334c568db2660ffc502b69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731521657010 2024-11-13T18:14:20,273 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e5b2bdd6f724bdb8898b4215eb540b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731521657710 2024-11-13T18:14:20,273 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3236b745d2b40f8a0bafb53e9c091a0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521658975 2024-11-13T18:14:20,275 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 24b2da40bfc94e419f03d5ec9888d7fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731521657010 2024-11-13T18:14:20,275 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 02928dabe32d446a8bcc69c71291cdf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731521657710 2024-11-13T18:14:20,276 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 7caa5894c3bf42f4b85c1a8ff6870499, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521658975 2024-11-13T18:14:20,295 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:20,296 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/c21edc29690b440c97362456cebaa5e7 is 50, key is test_row_0/A:col10/1731521658975/Put/seqid=0 2024-11-13T18:14:20,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T18:14:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:20,297 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:14:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:20,300 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:20,301 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/5092c21c9514433ab4f405d478fa6434 is 50, key is test_row_0/B:col10/1731521658975/Put/seqid=0 2024-11-13T18:14:20,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/fcb59e6983f34462a9bb1fe12af53bac is 50, key is test_row_0/A:col10/1731521659383/Put/seqid=0 2024-11-13T18:14:20,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742394_1570 (size=12629) 2024-11-13T18:14:20,330 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/c21edc29690b440c97362456cebaa5e7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c21edc29690b440c97362456cebaa5e7 2024-11-13T18:14:20,338 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into c21edc29690b440c97362456cebaa5e7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:20,338 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:20,338 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=12, startTime=1731521660269; duration=0sec 2024-11-13T18:14:20,338 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:20,339 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:20,339 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:20,341 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:20,341 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:20,341 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:20,341 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ed40bca2e08d426784fdab58cfbb5e33, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/13204f8d84e34fd3b366b6b604a4e6cf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/553edf81b563480fbdf25a9a455e6c75, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41e99a5c7c6e4ad2a15f2258e6378f72] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=47.8 K 2024-11-13T18:14:20,341 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed40bca2e08d426784fdab58cfbb5e33, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731521656834 2024-11-13T18:14:20,342 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13204f8d84e34fd3b366b6b604a4e6cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731521657010 2024-11-13T18:14:20,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742396_1572 (size=12151) 2024-11-13T18:14:20,344 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 553edf81b563480fbdf25a9a455e6c75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731521657710 2024-11-13T18:14:20,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742395_1571 (size=12629) 2024-11-13T18:14:20,344 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41e99a5c7c6e4ad2a15f2258e6378f72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521658975 2024-11-13T18:14:20,349 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/5092c21c9514433ab4f405d478fa6434 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5092c21c9514433ab4f405d478fa6434 2024-11-13T18:14:20,355 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into 5092c21c9514433ab4f405d478fa6434(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:20,356 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:20,356 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=12, startTime=1731521660269; duration=0sec 2024-11-13T18:14:20,356 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:20,356 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:20,359 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:20,359 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/dfb02d62bd1c46889638d4f697859e53 is 50, key is test_row_0/C:col10/1731521658975/Put/seqid=0 2024-11-13T18:14:20,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742397_1573 (size=12629) 2024-11-13T18:14:20,375 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/dfb02d62bd1c46889638d4f697859e53 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/dfb02d62bd1c46889638d4f697859e53 2024-11-13T18:14:20,379 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into dfb02d62bd1c46889638d4f697859e53(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:20,379 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:20,379 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=12, startTime=1731521660270; duration=0sec 2024-11-13T18:14:20,379 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:20,379 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:14:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:20,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:20,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521720590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521720591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521660572, max timestamp=1731521660602Num uniques 859; 2024-11-13T18:14:20,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521660572, max timestamp=1731521660602Num uniques 859; 2024-11-13T18:14:20,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:20,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:20,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521660572, max timestamp=1731521660602Num uniques 859; 2024-11-13T18:14:20,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:20,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521720657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521720659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521720660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521720714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521720714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,750 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/fcb59e6983f34462a9bb1fe12af53bac 2024-11-13T18:14:20,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/21083a16fcdd4b19904a8a5e5e68aaf6 is 50, key is test_row_0/B:col10/1731521659383/Put/seqid=0 2024-11-13T18:14:20,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742398_1574 (size=12151) 2024-11-13T18:14:20,783 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/21083a16fcdd4b19904a8a5e5e68aaf6 2024-11-13T18:14:20,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521720774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521720784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521720785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f656f02eb3a1483baf834089d9ef4df2 is 50, key is test_row_0/C:col10/1731521659383/Put/seqid=0 2024-11-13T18:14:20,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742399_1575 (size=12151) 2024-11-13T18:14:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521720924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521720924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521720994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521721004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521721004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,214 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f656f02eb3a1483baf834089d9ef4df2 2024-11-13T18:14:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/fcb59e6983f34462a9bb1fe12af53bac as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/fcb59e6983f34462a9bb1fe12af53bac 2024-11-13T18:14:21,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/fcb59e6983f34462a9bb1fe12af53bac, entries=150, sequenceid=235, filesize=11.9 K 2024-11-13T18:14:21,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/21083a16fcdd4b19904a8a5e5e68aaf6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/21083a16fcdd4b19904a8a5e5e68aaf6 2024-11-13T18:14:21,228 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/21083a16fcdd4b19904a8a5e5e68aaf6, entries=150, sequenceid=235, filesize=11.9 K 2024-11-13T18:14:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f656f02eb3a1483baf834089d9ef4df2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f656f02eb3a1483baf834089d9ef4df2 2024-11-13T18:14:21,232 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f656f02eb3a1483baf834089d9ef4df2, entries=150, sequenceid=235, filesize=11.9 K 2024-11-13T18:14:21,233 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 278d3a3ebd6be30bccd59b5fe89f892a in 936ms, sequenceid=235, compaction requested=false 2024-11-13T18:14:21,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:21,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:21,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-13T18:14:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-13T18:14:21,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-13T18:14:21,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8650 sec 2024-11-13T18:14:21,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8710 sec 2024-11-13T18:14:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:21,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T18:14:21,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:21,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:21,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:21,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:21,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:21,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:21,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/826c3a7e0f14474694b0ea055376ccf2 is 50, key is test_row_0/A:col10/1731521661260/Put/seqid=0 2024-11-13T18:14:21,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742400_1576 (size=14741) 2024-11-13T18:14:21,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521721304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521721308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521721310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521721324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521721324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521721428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521721434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:14:21,514 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:21,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-13T18:14:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:14:21,516 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:21,517 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:21,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:14:21,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521721655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521721655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/826c3a7e0f14474694b0ea055376ccf2 2024-11-13T18:14:21,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:14:21,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:21,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:21,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:21,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:21,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:21,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/7a9fa5588dfa4afd80ace47fb74f0a3b is 50, key is test_row_0/B:col10/1731521661260/Put/seqid=0 2024-11-13T18:14:21,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742401_1577 (size=12301) 2024-11-13T18:14:21,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521721828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:14:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:14:21,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:21,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:21,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:21,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:21,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:21,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521721845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521721845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521721964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521721964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:22,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:14:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/7a9fa5588dfa4afd80ace47fb74f0a3b 2024-11-13T18:14:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:14:22,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b79acdc0bc354dee9575867b1a0c2ecf is 50, key is test_row_0/C:col10/1731521661260/Put/seqid=0 2024-11-13T18:14:22,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742402_1578 (size=12301) 2024-11-13T18:14:22,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b79acdc0bc354dee9575867b1a0c2ecf 2024-11-13T18:14:22,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:14:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/826c3a7e0f14474694b0ea055376ccf2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826c3a7e0f14474694b0ea055376ccf2 2024-11-13T18:14:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826c3a7e0f14474694b0ea055376ccf2, entries=200, sequenceid=274, filesize=14.4 K 2024-11-13T18:14:22,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/7a9fa5588dfa4afd80ace47fb74f0a3b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7a9fa5588dfa4afd80ace47fb74f0a3b 2024-11-13T18:14:22,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7a9fa5588dfa4afd80ace47fb74f0a3b, entries=150, sequenceid=274, filesize=12.0 K 2024-11-13T18:14:22,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b79acdc0bc354dee9575867b1a0c2ecf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b79acdc0bc354dee9575867b1a0c2ecf 2024-11-13T18:14:22,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b79acdc0bc354dee9575867b1a0c2ecf, entries=150, sequenceid=274, filesize=12.0 K 2024-11-13T18:14:22,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 278d3a3ebd6be30bccd59b5fe89f892a in 922ms, sequenceid=274, compaction requested=true 2024-11-13T18:14:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:22,184 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:22,185 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:22,189 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:22,189 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:22,189 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,189 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c21edc29690b440c97362456cebaa5e7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/fcb59e6983f34462a9bb1fe12af53bac, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826c3a7e0f14474694b0ea055376ccf2] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=38.6 K 2024-11-13T18:14:22,192 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:22,192 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:22,192 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,192 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5092c21c9514433ab4f405d478fa6434, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/21083a16fcdd4b19904a8a5e5e68aaf6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7a9fa5588dfa4afd80ace47fb74f0a3b] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=36.2 K 2024-11-13T18:14:22,209 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5092c21c9514433ab4f405d478fa6434, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521658975 2024-11-13T18:14:22,209 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c21edc29690b440c97362456cebaa5e7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521658975 2024-11-13T18:14:22,211 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21083a16fcdd4b19904a8a5e5e68aaf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731521659364 2024-11-13T18:14:22,211 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb59e6983f34462a9bb1fe12af53bac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731521659364 2024-11-13T18:14:22,211 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 826c3a7e0f14474694b0ea055376ccf2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731521660650 2024-11-13T18:14:22,212 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a9fa5588dfa4afd80ace47fb74f0a3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731521660650 2024-11-13T18:14:22,222 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:22,222 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7d30dcc3000b4c419a625340f64afaf8 is 50, key is test_row_0/A:col10/1731521661260/Put/seqid=0 2024-11-13T18:14:22,235 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#489 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:22,236 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/b30d135c25834411b9ad67e724313439 is 50, key is test_row_0/B:col10/1731521661260/Put/seqid=0 2024-11-13T18:14:22,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742403_1579 (size=12881) 2024-11-13T18:14:22,246 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7d30dcc3000b4c419a625340f64afaf8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7d30dcc3000b4c419a625340f64afaf8 2024-11-13T18:14:22,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742404_1580 (size=12881) 2024-11-13T18:14:22,257 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into 7d30dcc3000b4c419a625340f64afaf8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:22,257 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:22,257 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=13, startTime=1731521662184; duration=0sec 2024-11-13T18:14:22,257 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:22,257 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:22,257 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:22,261 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:22,261 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:22,261 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,261 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/dfb02d62bd1c46889638d4f697859e53, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f656f02eb3a1483baf834089d9ef4df2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b79acdc0bc354dee9575867b1a0c2ecf] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=36.2 K 2024-11-13T18:14:22,261 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting dfb02d62bd1c46889638d4f697859e53, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731521658975 2024-11-13T18:14:22,262 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f656f02eb3a1483baf834089d9ef4df2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731521659364 2024-11-13T18:14:22,262 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b79acdc0bc354dee9575867b1a0c2ecf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731521660650 2024-11-13T18:14:22,268 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#490 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:22,269 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f3516d95ed3d4c558e7a9e33a0cea246 is 50, key is test_row_0/C:col10/1731521661260/Put/seqid=0 2024-11-13T18:14:22,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742405_1581 (size=12881) 2024-11-13T18:14:22,290 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f3516d95ed3d4c558e7a9e33a0cea246 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f3516d95ed3d4c558e7a9e33a0cea246 2024-11-13T18:14:22,294 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into f3516d95ed3d4c558e7a9e33a0cea246(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:22,294 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:22,294 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=13, startTime=1731521662184; duration=0sec 2024-11-13T18:14:22,294 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:22,294 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:22,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:14:22,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,322 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:14:22,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:22,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:22,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/4499eb2bd5214dd9b8dd190a94f1e330 is 50, key is test_row_0/A:col10/1731521661307/Put/seqid=0 2024-11-13T18:14:22,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742406_1582 (size=12301) 2024-11-13T18:14:22,348 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/4499eb2bd5214dd9b8dd190a94f1e330 2024-11-13T18:14:22,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6a4999a3f13d4ec59c515d378c122916 is 50, key is test_row_0/B:col10/1731521661307/Put/seqid=0 2024-11-13T18:14:22,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742407_1583 (size=12301) 2024-11-13T18:14:22,411 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6a4999a3f13d4ec59c515d378c122916 2024-11-13T18:14:22,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/12389b814f7e45f0b5503e5683caceeb is 50, key is test_row_0/C:col10/1731521661307/Put/seqid=0 2024-11-13T18:14:22,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742408_1584 (size=12301) 2024-11-13T18:14:22,443 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/12389b814f7e45f0b5503e5683caceeb 2024-11-13T18:14:22,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/4499eb2bd5214dd9b8dd190a94f1e330 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/4499eb2bd5214dd9b8dd190a94f1e330 2024-11-13T18:14:22,480 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/4499eb2bd5214dd9b8dd190a94f1e330, entries=150, sequenceid=286, filesize=12.0 K 2024-11-13T18:14:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6a4999a3f13d4ec59c515d378c122916 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6a4999a3f13d4ec59c515d378c122916 2024-11-13T18:14:22,485 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6a4999a3f13d4ec59c515d378c122916, entries=150, sequenceid=286, filesize=12.0 K 2024-11-13T18:14:22,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/12389b814f7e45f0b5503e5683caceeb as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/12389b814f7e45f0b5503e5683caceeb 2024-11-13T18:14:22,490 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/12389b814f7e45f0b5503e5683caceeb, entries=150, sequenceid=286, filesize=12.0 K 2024-11-13T18:14:22,491 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 278d3a3ebd6be30bccd59b5fe89f892a in 169ms, sequenceid=286, compaction requested=false 2024-11-13T18:14:22,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:22,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-13T18:14:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-13T18:14:22,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-13T18:14:22,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 976 msec 2024-11-13T18:14:22,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 981 msec 2024-11-13T18:14:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:22,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:22,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/853c7bb3bcb24adbb48b03f24bd0be24 is 50, key is test_row_0/A:col10/1731521662494/Put/seqid=0 2024-11-13T18:14:22,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742409_1585 (size=14741) 2024-11-13T18:14:22,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521662548, max timestamp=1731521662605Num uniques 859; 2024-11-13T18:14:22,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:22,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521662548, max timestamp=1731521662606Num uniques 859; 2024-11-13T18:14:22,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:22,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521662548, max timestamp=1731521662605Num uniques 859; 2024-11-13T18:14:22,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:22,652 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/b30d135c25834411b9ad67e724313439 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b30d135c25834411b9ad67e724313439 2024-11-13T18:14:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:14:22,654 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:22,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-13T18:14:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:14:22,658 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into b30d135c25834411b9ad67e724313439(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:22,658 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:22,658 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=13, startTime=1731521662184; duration=0sec 2024-11-13T18:14:22,659 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:22,659 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:22,659 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:22,660 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:22,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:22,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521722734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:22,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:22,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521722737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:14:22,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521722845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521722845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:22,857 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 278d3a3ebd6be30bccd59b5fe89f892a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:22,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521722854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:22,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521722864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:22,877 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 278d3a3ebd6be30bccd59b5fe89f892a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:22,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521722874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:22,886 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_2' in 278d3a3ebd6be30bccd59b5fe89f892a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:22,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/853c7bb3bcb24adbb48b03f24bd0be24 2024-11-13T18:14:22,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6105fb61a41644e4b8d694b6645dba62 is 50, key is test_row_0/B:col10/1731521662494/Put/seqid=0 2024-11-13T18:14:22,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742410_1586 (size=12301) 2024-11-13T18:14:22,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:22,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:14:23,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521723074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:23,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521723074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:23,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:23,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:23,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:14:23,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6105fb61a41644e4b8d694b6645dba62 2024-11-13T18:14:23,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/ad10b6abc5d446cdaed30b1eafdb4053 is 50, key is test_row_0/C:col10/1731521662494/Put/seqid=0 2024-11-13T18:14:23,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742411_1587 (size=12301) 2024-11-13T18:14:23,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/ad10b6abc5d446cdaed30b1eafdb4053 2024-11-13T18:14:23,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/853c7bb3bcb24adbb48b03f24bd0be24 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/853c7bb3bcb24adbb48b03f24bd0be24 2024-11-13T18:14:23,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/853c7bb3bcb24adbb48b03f24bd0be24, entries=200, sequenceid=296, filesize=14.4 K 2024-11-13T18:14:23,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6105fb61a41644e4b8d694b6645dba62 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6105fb61a41644e4b8d694b6645dba62 2024-11-13T18:14:23,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6105fb61a41644e4b8d694b6645dba62, entries=150, sequenceid=296, filesize=12.0 K 2024-11-13T18:14:23,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/ad10b6abc5d446cdaed30b1eafdb4053 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ad10b6abc5d446cdaed30b1eafdb4053 2024-11-13T18:14:23,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ad10b6abc5d446cdaed30b1eafdb4053, entries=150, sequenceid=296, filesize=12.0 K 2024-11-13T18:14:23,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 278d3a3ebd6be30bccd59b5fe89f892a in 852ms, sequenceid=296, compaction requested=true 2024-11-13T18:14:23,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:23,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:23,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:23,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:23,362 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:23,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:23,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:23,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:23,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:23,365 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:23,365 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:23,365 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:23,365 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:23,365 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,365 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,366 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7d30dcc3000b4c419a625340f64afaf8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/4499eb2bd5214dd9b8dd190a94f1e330, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/853c7bb3bcb24adbb48b03f24bd0be24] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=39.0 K 2024-11-13T18:14:23,366 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b30d135c25834411b9ad67e724313439, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6a4999a3f13d4ec59c515d378c122916, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6105fb61a41644e4b8d694b6645dba62] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=36.6 K 2024-11-13T18:14:23,366 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d30dcc3000b4c419a625340f64afaf8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731521660650 2024-11-13T18:14:23,366 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b30d135c25834411b9ad67e724313439, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731521660650 2024-11-13T18:14:23,366 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a4999a3f13d4ec59c515d378c122916, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731521661279 2024-11-13T18:14:23,366 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4499eb2bd5214dd9b8dd190a94f1e330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731521661279 2024-11-13T18:14:23,367 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6105fb61a41644e4b8d694b6645dba62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731521662490 2024-11-13T18:14:23,367 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 853c7bb3bcb24adbb48b03f24bd0be24, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731521662490 2024-11-13T18:14:23,374 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:23,375 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/6fd044c714f24fdc983816c0b2d883c3 is 50, key is test_row_0/A:col10/1731521662494/Put/seqid=0 2024-11-13T18:14:23,375 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:23,376 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/ee684167db664ce190e41b67e46f513d is 50, key is test_row_0/B:col10/1731521662494/Put/seqid=0 2024-11-13T18:14:23,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742412_1588 (size=12983) 2024-11-13T18:14:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:23,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521662698, max timestamp=1731521663403Num uniques 1018; 2024-11-13T18:14:23,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:14:23,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521662698, max timestamp=1731521663403Num uniques 1018; 2024-11-13T18:14:23,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521662698, max timestamp=1731521663403Num uniques 1018; 2024-11-13T18:14:23,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:14:23,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:14:23,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-11-13T18:14:23,416 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/ee684167db664ce190e41b67e46f513d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ee684167db664ce190e41b67e46f513d 2024-11-13T18:14:23,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:23,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:23,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:23,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:23,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:23,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:23,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742413_1589 (size=12983) 2024-11-13T18:14:23,424 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into ee684167db664ce190e41b67e46f513d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:23,424 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:23,424 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=13, startTime=1731521663362; duration=0sec 2024-11-13T18:14:23,424 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:23,424 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:23,424 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:23,425 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:23,425 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:23,425 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,425 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f3516d95ed3d4c558e7a9e33a0cea246, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/12389b814f7e45f0b5503e5683caceeb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ad10b6abc5d446cdaed30b1eafdb4053] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=36.6 K 2024-11-13T18:14:23,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/1b169efc7f33465f83c8cc23d5290092 is 50, key is test_row_0/A:col10/1731521663412/Put/seqid=0 2024-11-13T18:14:23,427 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f3516d95ed3d4c558e7a9e33a0cea246, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731521660650 2024-11-13T18:14:23,430 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 12389b814f7e45f0b5503e5683caceeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731521661279 2024-11-13T18:14:23,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:23,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:23,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,433 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ad10b6abc5d446cdaed30b1eafdb4053, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731521662490 2024-11-13T18:14:23,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,452 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:23,452 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/6017f5051a5b41f58d8a89fb519be1d3 is 50, key is test_row_0/C:col10/1731521662494/Put/seqid=0 2024-11-13T18:14:23,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742414_1590 (size=17181) 2024-11-13T18:14:23,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742415_1591 (size=12983) 2024-11-13T18:14:23,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/1b169efc7f33465f83c8cc23d5290092 2024-11-13T18:14:23,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/67e0d515d51a4bb6ba08fcbbfcd73e9a is 50, key is test_row_0/B:col10/1731521663412/Put/seqid=0 2024-11-13T18:14:23,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742416_1592 (size=12301) 2024-11-13T18:14:23,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/67e0d515d51a4bb6ba08fcbbfcd73e9a 2024-11-13T18:14:23,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/caf95196d3e2497a903f8f5fbc3ec21a is 50, key is test_row_0/C:col10/1731521663412/Put/seqid=0 2024-11-13T18:14:23,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742417_1593 (size=12301) 2024-11-13T18:14:23,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731521663489, max timestamp=1731521663559Num uniques 859; 2024-11-13T18:14:23,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:14:23,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731521663489, max timestamp=1731521663559Num uniques 859; 2024-11-13T18:14:23,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:14:23,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731521663489, max timestamp=1731521663559Num uniques 859; 2024-11-13T18:14:23,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:14:23,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521723697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:23,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521663664, max timestamp=1731521663710Num uniques 1118; 2024-11-13T18:14:23,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521663664, max timestamp=1731521663710Num uniques 1118; 2024-11-13T18:14:23,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-13T18:14:23,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-13T18:14:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:23,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521663664, max timestamp=1731521663710Num uniques 1118; 2024-11-13T18:14:23,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-13T18:14:23,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:23,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:23,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:14:23,830 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/6fd044c714f24fdc983816c0b2d883c3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6fd044c714f24fdc983816c0b2d883c3 2024-11-13T18:14:23,835 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into 6fd044c714f24fdc983816c0b2d883c3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:23,835 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:23,835 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=13, startTime=1731521663362; duration=0sec 2024-11-13T18:14:23,835 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:23,835 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:23,880 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/6017f5051a5b41f58d8a89fb519be1d3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/6017f5051a5b41f58d8a89fb519be1d3 2024-11-13T18:14:23,887 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into 6017f5051a5b41f58d8a89fb519be1d3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:23,887 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:23,887 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=13, startTime=1731521663363; duration=0sec 2024-11-13T18:14:23,887 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:23,887 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:23,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:23,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:23,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/caf95196d3e2497a903f8f5fbc3ec21a 2024-11-13T18:14:23,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731521663902, max timestamp=1731521663940Num uniques 1027; 2024-11-13T18:14:23,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731521663902, max timestamp=1731521663940Num uniques 1027; 2024-11-13T18:14:23,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T18:14:23,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T18:14:23,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731521663902, max timestamp=1731521663940Num uniques 1027; 2024-11-13T18:14:23,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T18:14:23,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/1b169efc7f33465f83c8cc23d5290092 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1b169efc7f33465f83c8cc23d5290092 2024-11-13T18:14:23,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1b169efc7f33465f83c8cc23d5290092, entries=250, sequenceid=336, filesize=16.8 K 2024-11-13T18:14:23,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/67e0d515d51a4bb6ba08fcbbfcd73e9a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/67e0d515d51a4bb6ba08fcbbfcd73e9a 2024-11-13T18:14:23,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/67e0d515d51a4bb6ba08fcbbfcd73e9a, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T18:14:23,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/caf95196d3e2497a903f8f5fbc3ec21a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/caf95196d3e2497a903f8f5fbc3ec21a 2024-11-13T18:14:23,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/caf95196d3e2497a903f8f5fbc3ec21a, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T18:14:23,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=53.67 KB/54960 for 278d3a3ebd6be30bccd59b5fe89f892a in 552ms, sequenceid=336, compaction requested=false 2024-11-13T18:14:23,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:23,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=60.38 KB heapSize=159.47 KB 2024-11-13T18:14:23,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:23,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:23,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:23,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:23,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:23,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:24,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7ca628d5027847699bd6e4880b1b64a6 is 50, key is test_row_0/A:col10/1731521663969/Put/seqid=0 2024-11-13T18:14:24,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742418_1594 (size=12301) 2024-11-13T18:14:24,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:24,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:24,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7ca628d5027847699bd6e4880b1b64a6 2024-11-13T18:14:24,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6ee1c6a26f5044b7b772566a3a1c5579 is 50, key is test_row_0/B:col10/1731521663969/Put/seqid=0 2024-11-13T18:14:24,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742419_1595 (size=12301) 2024-11-13T18:14:24,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731521664111, max timestamp=1731521664157Num uniques 859; 2024-11-13T18:14:24,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:24,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521664111, max timestamp=1731521664157Num uniques 859; 2024-11-13T18:14:24,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:24,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521664111, max timestamp=1731521664157Num uniques 859; 2024-11-13T18:14:24,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:24,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521724249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521724262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521724374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521724374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6ee1c6a26f5044b7b772566a3a1c5579 2024-11-13T18:14:24,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/123e2ae3194549b48fd789b6bfba6b74 is 50, key is test_row_0/C:col10/1731521663969/Put/seqid=0 2024-11-13T18:14:24,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742420_1596 (size=12301) 2024-11-13T18:14:24,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:24,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:24,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521724584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521724584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:24,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:24,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:14:24,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:24,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:24,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521724865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,880 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 278d3a3ebd6be30bccd59b5fe89f892a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/123e2ae3194549b48fd789b6bfba6b74 2024-11-13T18:14:24,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7ca628d5027847699bd6e4880b1b64a6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7ca628d5027847699bd6e4880b1b64a6 2024-11-13T18:14:24,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7ca628d5027847699bd6e4880b1b64a6, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T18:14:24,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/6ee1c6a26f5044b7b772566a3a1c5579 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6ee1c6a26f5044b7b772566a3a1c5579 2024-11-13T18:14:24,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6ee1c6a26f5044b7b772566a3a1c5579, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T18:14:24,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/123e2ae3194549b48fd789b6bfba6b74 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/123e2ae3194549b48fd789b6bfba6b74 2024-11-13T18:14:24,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521724895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521724896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,912 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 278d3a3ebd6be30bccd59b5fe89f892a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:24,912 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_2' in 278d3a3ebd6be30bccd59b5fe89f892a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521724914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521724914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:24,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/123e2ae3194549b48fd789b6bfba6b74, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T18:14:24,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~159.42 KB/163248, currentSize=127.47 KB/130530 for 278d3a3ebd6be30bccd59b5fe89f892a in 956ms, sequenceid=398, compaction requested=true 2024-11-13T18:14:24,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:24,928 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:24,928 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:24,935 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:24,935 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:24,935 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,935 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ee684167db664ce190e41b67e46f513d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/67e0d515d51a4bb6ba08fcbbfcd73e9a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6ee1c6a26f5044b7b772566a3a1c5579] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=36.7 K 2024-11-13T18:14:24,935 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ee684167db664ce190e41b67e46f513d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731521662490 2024-11-13T18:14:24,936 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e0d515d51a4bb6ba08fcbbfcd73e9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731521662698 2024-11-13T18:14:24,935 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:24,936 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:24,936 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,937 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6fd044c714f24fdc983816c0b2d883c3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1b169efc7f33465f83c8cc23d5290092, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7ca628d5027847699bd6e4880b1b64a6] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=41.5 K 2024-11-13T18:14:24,937 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee1c6a26f5044b7b772566a3a1c5579, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731521663902 2024-11-13T18:14:24,937 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fd044c714f24fdc983816c0b2d883c3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731521662490 2024-11-13T18:14:24,938 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b169efc7f33465f83c8cc23d5290092, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731521662698 2024-11-13T18:14:24,939 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ca628d5027847699bd6e4880b1b64a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731521663902 2024-11-13T18:14:24,946 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#506 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:24,946 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/70fe8ef61e364714a57e8a19fab7eef1 is 50, key is test_row_0/B:col10/1731521663969/Put/seqid=0 2024-11-13T18:14:24,949 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:24,950 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/c2ce5ae6c37f467aa13a6999c7c2bc20 is 50, key is test_row_0/A:col10/1731521663969/Put/seqid=0 2024-11-13T18:14:24,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:14:24,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:24,979 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T18:14:24,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:24,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:24,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:24,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:24,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:24,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:24,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/f032b0483b2445f98f2e1daf23c73c50 is 50, key is test_row_0/A:col10/1731521664228/Put/seqid=0 2024-11-13T18:14:24,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742422_1598 (size=13085) 2024-11-13T18:14:24,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742421_1597 (size=13085) 2024-11-13T18:14:25,002 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/c2ce5ae6c37f467aa13a6999c7c2bc20 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c2ce5ae6c37f467aa13a6999c7c2bc20 2024-11-13T18:14:25,012 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into c2ce5ae6c37f467aa13a6999c7c2bc20(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:25,012 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:25,012 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=13, startTime=1731521664928; duration=0sec 2024-11-13T18:14:25,012 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:25,012 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:25,012 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:25,017 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:25,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742423_1599 (size=12301) 2024-11-13T18:14:25,017 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:25,017 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:25,017 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/6017f5051a5b41f58d8a89fb519be1d3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/caf95196d3e2497a903f8f5fbc3ec21a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/123e2ae3194549b48fd789b6bfba6b74] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=36.7 K 2024-11-13T18:14:25,019 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6017f5051a5b41f58d8a89fb519be1d3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731521662490 2024-11-13T18:14:25,020 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting caf95196d3e2497a903f8f5fbc3ec21a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731521662698 2024-11-13T18:14:25,020 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 123e2ae3194549b48fd789b6bfba6b74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731521663902 2024-11-13T18:14:25,020 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/f032b0483b2445f98f2e1daf23c73c50 2024-11-13T18:14:25,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/e13ecc72f9be4046bcf7a9d39e0b2cd9 is 50, key is test_row_0/B:col10/1731521664228/Put/seqid=0 2024-11-13T18:14:25,046 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:25,047 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/739c39b7a73845da8a69fb3bd319343d is 50, key is test_row_0/C:col10/1731521663969/Put/seqid=0 2024-11-13T18:14:25,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742424_1600 (size=12301) 2024-11-13T18:14:25,081 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/e13ecc72f9be4046bcf7a9d39e0b2cd9 2024-11-13T18:14:25,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/3288390634194c54bad05e057771bc81 is 50, key is test_row_0/C:col10/1731521664228/Put/seqid=0 2024-11-13T18:14:25,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742425_1601 (size=13085) 2024-11-13T18:14:25,114 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/739c39b7a73845da8a69fb3bd319343d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/739c39b7a73845da8a69fb3bd319343d 2024-11-13T18:14:25,118 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into 739c39b7a73845da8a69fb3bd319343d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:25,118 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:25,118 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=13, startTime=1731521664928; duration=0sec 2024-11-13T18:14:25,118 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:25,118 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:25,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742426_1602 (size=12301) 2024-11-13T18:14:25,406 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/70fe8ef61e364714a57e8a19fab7eef1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/70fe8ef61e364714a57e8a19fab7eef1 2024-11-13T18:14:25,411 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into 70fe8ef61e364714a57e8a19fab7eef1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:25,411 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:25,411 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=13, startTime=1731521664928; duration=0sec 2024-11-13T18:14:25,412 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:25,412 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:25,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:25,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521725485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:25,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521725488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:25,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:25,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:25,529 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/3288390634194c54bad05e057771bc81 2024-11-13T18:14:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/f032b0483b2445f98f2e1daf23c73c50 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f032b0483b2445f98f2e1daf23c73c50 2024-11-13T18:14:25,538 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f032b0483b2445f98f2e1daf23c73c50, entries=150, sequenceid=433, filesize=12.0 K 2024-11-13T18:14:25,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/e13ecc72f9be4046bcf7a9d39e0b2cd9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e13ecc72f9be4046bcf7a9d39e0b2cd9 2024-11-13T18:14:25,543 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e13ecc72f9be4046bcf7a9d39e0b2cd9, entries=150, sequenceid=433, filesize=12.0 K 2024-11-13T18:14:25,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/3288390634194c54bad05e057771bc81 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3288390634194c54bad05e057771bc81 2024-11-13T18:14:25,551 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3288390634194c54bad05e057771bc81, entries=150, sequenceid=433, filesize=12.0 K 2024-11-13T18:14:25,557 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 278d3a3ebd6be30bccd59b5fe89f892a in 578ms, sequenceid=433, compaction requested=false 2024-11-13T18:14:25,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:25,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:25,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-13T18:14:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-13T18:14:25,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-13T18:14:25,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8980 sec 2024-11-13T18:14:25,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.9070 sec 2024-11-13T18:14:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:25,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:14:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:25,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/41241cc2395148848a6a8a8cc5e4a0ac is 50, key is test_row_0/A:col10/1731521665613/Put/seqid=0 2024-11-13T18:14:25,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742427_1603 (size=12301) 2024-11-13T18:14:25,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521725718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:25,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521725725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:25,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521725848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:25,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521725848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:26,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521726064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:26,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521726065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:26,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/41241cc2395148848a6a8a8cc5e4a0ac 2024-11-13T18:14:26,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/25edf1f53504415e9dff70a873177c77 is 50, key is test_row_0/B:col10/1731521665613/Put/seqid=0 2024-11-13T18:14:26,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742428_1604 (size=12301) 2024-11-13T18:14:26,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/25edf1f53504415e9dff70a873177c77 2024-11-13T18:14:26,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/a6ea65c536d14e378d4c25b37853c9c8 is 50, key is test_row_0/C:col10/1731521665613/Put/seqid=0 2024-11-13T18:14:26,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742429_1605 (size=12301) 2024-11-13T18:14:26,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/a6ea65c536d14e378d4c25b37853c9c8 2024-11-13T18:14:26,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/41241cc2395148848a6a8a8cc5e4a0ac as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/41241cc2395148848a6a8a8cc5e4a0ac 2024-11-13T18:14:26,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/41241cc2395148848a6a8a8cc5e4a0ac, entries=150, sequenceid=449, filesize=12.0 K 2024-11-13T18:14:26,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/25edf1f53504415e9dff70a873177c77 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/25edf1f53504415e9dff70a873177c77 2024-11-13T18:14:26,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/25edf1f53504415e9dff70a873177c77, entries=150, sequenceid=449, filesize=12.0 K 2024-11-13T18:14:26,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/a6ea65c536d14e378d4c25b37853c9c8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a6ea65c536d14e378d4c25b37853c9c8 2024-11-13T18:14:26,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a6ea65c536d14e378d4c25b37853c9c8, entries=150, sequenceid=449, filesize=12.0 K 2024-11-13T18:14:26,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 278d3a3ebd6be30bccd59b5fe89f892a in 680ms, sequenceid=449, compaction requested=true 2024-11-13T18:14:26,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:26,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:14:26,294 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:26,294 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:26,295 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:26,295 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:26,295 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:26,295 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/739c39b7a73845da8a69fb3bd319343d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3288390634194c54bad05e057771bc81, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a6ea65c536d14e378d4c25b37853c9c8] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=36.8 K 2024-11-13T18:14:26,295 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:26,295 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:26,295 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:26,295 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c2ce5ae6c37f467aa13a6999c7c2bc20, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f032b0483b2445f98f2e1daf23c73c50, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/41241cc2395148848a6a8a8cc5e4a0ac] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=36.8 K 2024-11-13T18:14:26,298 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 739c39b7a73845da8a69fb3bd319343d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731521663902 2024-11-13T18:14:26,298 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ce5ae6c37f467aa13a6999c7c2bc20, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731521663902 2024-11-13T18:14:26,298 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f032b0483b2445f98f2e1daf23c73c50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731521664228 2024-11-13T18:14:26,298 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 3288390634194c54bad05e057771bc81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731521664228 2024-11-13T18:14:26,299 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41241cc2395148848a6a8a8cc5e4a0ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731521665486 2024-11-13T18:14:26,299 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting a6ea65c536d14e378d4c25b37853c9c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731521665486 2024-11-13T18:14:26,323 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:26,323 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/7b69e67612f14f53aaf7d1701cb93ac4 is 50, key is test_row_0/C:col10/1731521665613/Put/seqid=0 2024-11-13T18:14:26,326 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:26,327 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/9e069843ba374797929b1d84065a4de1 is 50, key is test_row_0/A:col10/1731521665613/Put/seqid=0 2024-11-13T18:14:26,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742430_1606 (size=13187) 2024-11-13T18:14:26,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742431_1607 (size=13187) 2024-11-13T18:14:26,348 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/7b69e67612f14f53aaf7d1701cb93ac4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/7b69e67612f14f53aaf7d1701cb93ac4 2024-11-13T18:14:26,370 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into 7b69e67612f14f53aaf7d1701cb93ac4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:26,370 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:26,370 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=13, startTime=1731521666294; duration=0sec 2024-11-13T18:14:26,370 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:26,370 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:26,370 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:26,376 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:26,376 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:26,377 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:26,377 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/70fe8ef61e364714a57e8a19fab7eef1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e13ecc72f9be4046bcf7a9d39e0b2cd9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/25edf1f53504415e9dff70a873177c77] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=36.8 K 2024-11-13T18:14:26,377 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 70fe8ef61e364714a57e8a19fab7eef1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731521663902 2024-11-13T18:14:26,380 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e13ecc72f9be4046bcf7a9d39e0b2cd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731521664228 2024-11-13T18:14:26,381 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 25edf1f53504415e9dff70a873177c77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731521665486 2024-11-13T18:14:26,395 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:26,395 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/49097970024747818780b9688cc3f86a is 50, key is test_row_0/B:col10/1731521665613/Put/seqid=0 2024-11-13T18:14:26,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:14:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:26,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:26,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:26,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:26,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:26,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:26,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:26,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742432_1608 (size=13187) 2024-11-13T18:14:26,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/65779f7185c7442290d515b6c926e5ec is 50, key is test_row_0/A:col10/1731521666402/Put/seqid=0 2024-11-13T18:14:26,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742433_1609 (size=14741) 2024-11-13T18:14:26,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521726484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:26,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521726503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:26,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:26,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521726614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:26,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521726637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:26,746 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/9e069843ba374797929b1d84065a4de1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/9e069843ba374797929b1d84065a4de1 2024-11-13T18:14:26,756 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into 9e069843ba374797929b1d84065a4de1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:26,756 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:26,756 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=13, startTime=1731521666293; duration=0sec 2024-11-13T18:14:26,756 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:26,756 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:14:26,824 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:26,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-13T18:14:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:14:26,838 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:26,840 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:26,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:26,841 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/49097970024747818780b9688cc3f86a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/49097970024747818780b9688cc3f86a 2024-11-13T18:14:26,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521726836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:26,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/65779f7185c7442290d515b6c926e5ec 2024-11-13T18:14:26,868 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into 49097970024747818780b9688cc3f86a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:26,868 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:26,869 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=13, startTime=1731521666294; duration=0sec 2024-11-13T18:14:26,869 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:26,869 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:26,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521726862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:26,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/cc6fefd6c0d94e108b2ed38e94959c2f is 50, key is test_row_0/B:col10/1731521666402/Put/seqid=0 2024-11-13T18:14:26,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742434_1610 (size=12301) 2024-11-13T18:14:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:14:26,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T18:14:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:26,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:27,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T18:14:27,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:27,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:27,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:27,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:14:27,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521727158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:27,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521727188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:27,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T18:14:27,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:27,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:27,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:27,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:27,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/cc6fefd6c0d94e108b2ed38e94959c2f 2024-11-13T18:14:27,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b9569645a2fb4b63bcfceca5e4799645 is 50, key is test_row_0/C:col10/1731521666402/Put/seqid=0 2024-11-13T18:14:27,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742435_1611 (size=12301) 2024-11-13T18:14:27,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b9569645a2fb4b63bcfceca5e4799645 2024-11-13T18:14:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/65779f7185c7442290d515b6c926e5ec as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/65779f7185c7442290d515b6c926e5ec 2024-11-13T18:14:27,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/65779f7185c7442290d515b6c926e5ec, entries=200, sequenceid=471, filesize=14.4 K 2024-11-13T18:14:27,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/cc6fefd6c0d94e108b2ed38e94959c2f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/cc6fefd6c0d94e108b2ed38e94959c2f 2024-11-13T18:14:27,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/cc6fefd6c0d94e108b2ed38e94959c2f, entries=150, sequenceid=471, filesize=12.0 K 2024-11-13T18:14:27,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b9569645a2fb4b63bcfceca5e4799645 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b9569645a2fb4b63bcfceca5e4799645 2024-11-13T18:14:27,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b9569645a2fb4b63bcfceca5e4799645, entries=150, sequenceid=471, filesize=12.0 K 2024-11-13T18:14:27,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 278d3a3ebd6be30bccd59b5fe89f892a in 1004ms, sequenceid=471, compaction requested=false 2024-11-13T18:14:27,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:14:27,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T18:14:27,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:27,469 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:14:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:27,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/301723cab32b4ac7bc39553fdba223d4 is 50, key is test_row_0/A:col10/1731521666495/Put/seqid=0 2024-11-13T18:14:27,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742436_1612 (size=12301) 2024-11-13T18:14:27,511 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/301723cab32b4ac7bc39553fdba223d4 2024-11-13T18:14:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/c6697294be4b4d1fb7c7e94ed7cb9ab0 is 50, key is test_row_0/B:col10/1731521666495/Put/seqid=0 2024-11-13T18:14:27,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742437_1613 (size=12301) 2024-11-13T18:14:27,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:27,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521667743, max timestamp=1731521667847Num uniques 859; 2024-11-13T18:14:27,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:27,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521667743, max timestamp=1731521667847Num uniques 859; 2024-11-13T18:14:27,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:14:27,993 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/c6697294be4b4d1fb7c7e94ed7cb9ab0 2024-11-13T18:14:28,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521727980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521727996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/31ca373cbd5c43c391aed6a59a447f26 is 50, key is test_row_0/C:col10/1731521666495/Put/seqid=0 2024-11-13T18:14:28,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742438_1614 (size=12301) 2024-11-13T18:14:28,062 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/31ca373cbd5c43c391aed6a59a447f26 2024-11-13T18:14:28,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/301723cab32b4ac7bc39553fdba223d4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/301723cab32b4ac7bc39553fdba223d4 2024-11-13T18:14:28,071 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/301723cab32b4ac7bc39553fdba223d4, entries=150, sequenceid=484, filesize=12.0 K 2024-11-13T18:14:28,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/c6697294be4b4d1fb7c7e94ed7cb9ab0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c6697294be4b4d1fb7c7e94ed7cb9ab0 2024-11-13T18:14:28,095 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c6697294be4b4d1fb7c7e94ed7cb9ab0, entries=150, sequenceid=484, filesize=12.0 K 2024-11-13T18:14:28,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/31ca373cbd5c43c391aed6a59a447f26 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/31ca373cbd5c43c391aed6a59a447f26 2024-11-13T18:14:28,111 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/31ca373cbd5c43c391aed6a59a447f26, entries=150, sequenceid=484, filesize=12.0 K 2024-11-13T18:14:28,112 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for 278d3a3ebd6be30bccd59b5fe89f892a in 643ms, sequenceid=484, compaction requested=true 2024-11-13T18:14:28,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:28,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:28,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-13T18:14:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-13T18:14:28,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-13T18:14:28,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2800 sec 2024-11-13T18:14:28,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.2940 sec 2024-11-13T18:14:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:28,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=129.71 KB heapSize=373.55 KB 2024-11-13T18:14:28,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:28,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:28,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:28,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:28,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:28,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:28,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/46993f4863c440f2abc2b85b405a0c23 is 50, key is test_row_0/A:col10/1731521667937/Put/seqid=0 2024-11-13T18:14:28,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742439_1615 (size=14741) 2024-11-13T18:14:28,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/46993f4863c440f2abc2b85b405a0c23 2024-11-13T18:14:28,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/69189ad9ae2340f793b53463c926b570 is 50, key is test_row_0/B:col10/1731521667937/Put/seqid=0 2024-11-13T18:14:28,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521728212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521728214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742440_1616 (size=12301) 2024-11-13T18:14:28,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/69189ad9ae2340f793b53463c926b570 2024-11-13T18:14:28,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/a287c903bebd4c79b425170a7b641049 is 50, key is test_row_0/C:col10/1731521667937/Put/seqid=0 2024-11-13T18:14:28,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742441_1617 (size=12301) 2024-11-13T18:14:28,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/a287c903bebd4c79b425170a7b641049 2024-11-13T18:14:28,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/46993f4863c440f2abc2b85b405a0c23 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/46993f4863c440f2abc2b85b405a0c23 2024-11-13T18:14:28,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/46993f4863c440f2abc2b85b405a0c23, entries=200, sequenceid=515, filesize=14.4 K 2024-11-13T18:14:28,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/69189ad9ae2340f793b53463c926b570 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/69189ad9ae2340f793b53463c926b570 2024-11-13T18:14:28,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521728336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/69189ad9ae2340f793b53463c926b570, entries=150, sequenceid=515, filesize=12.0 K 2024-11-13T18:14:28,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/a287c903bebd4c79b425170a7b641049 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a287c903bebd4c79b425170a7b641049 2024-11-13T18:14:28,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521728344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a287c903bebd4c79b425170a7b641049, entries=150, sequenceid=515, filesize=12.0 K 2024-11-13T18:14:28,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~373.50 KB/382464, currentSize=46.96 KB/48090 for 278d3a3ebd6be30bccd59b5fe89f892a in 229ms, sequenceid=515, compaction requested=true 2024-11-13T18:14:28,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:28,366 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:28,367 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:28,367 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:28,367 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:28,367 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/9e069843ba374797929b1d84065a4de1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/65779f7185c7442290d515b6c926e5ec, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/301723cab32b4ac7bc39553fdba223d4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/46993f4863c440f2abc2b85b405a0c23] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=53.7 K 2024-11-13T18:14:28,368 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e069843ba374797929b1d84065a4de1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731521665486 2024-11-13T18:14:28,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:28,368 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65779f7185c7442290d515b6c926e5ec, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731521665705 2024-11-13T18:14:28,368 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 301723cab32b4ac7bc39553fdba223d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731521666450 2024-11-13T18:14:28,369 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46993f4863c440f2abc2b85b405a0c23, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731521667937 2024-11-13T18:14:28,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:28,370 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:28,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:28,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:28,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:28,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:28,373 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:28,373 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:28,373 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:28,374 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/49097970024747818780b9688cc3f86a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/cc6fefd6c0d94e108b2ed38e94959c2f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c6697294be4b4d1fb7c7e94ed7cb9ab0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/69189ad9ae2340f793b53463c926b570] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=48.9 K 2024-11-13T18:14:28,374 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 49097970024747818780b9688cc3f86a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731521665486 2024-11-13T18:14:28,375 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting cc6fefd6c0d94e108b2ed38e94959c2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731521665723 2024-11-13T18:14:28,375 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c6697294be4b4d1fb7c7e94ed7cb9ab0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731521666450 2024-11-13T18:14:28,375 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 69189ad9ae2340f793b53463c926b570, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731521667937 2024-11-13T18:14:28,410 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:28,410 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/d23125d92c7b4e1c9643295d185052c6 is 50, key is test_row_0/A:col10/1731521667937/Put/seqid=0 2024-11-13T18:14:28,422 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#528 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:28,423 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/c9362673870c4a65a1cb34b2462a6e07 is 50, key is test_row_0/B:col10/1731521667937/Put/seqid=0 2024-11-13T18:14:28,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742442_1618 (size=13323) 2024-11-13T18:14:28,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742443_1619 (size=13323) 2024-11-13T18:14:28,507 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/d23125d92c7b4e1c9643295d185052c6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d23125d92c7b4e1c9643295d185052c6 2024-11-13T18:14:28,513 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into d23125d92c7b4e1c9643295d185052c6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:28,514 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:28,514 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=12, startTime=1731521668366; duration=0sec 2024-11-13T18:14:28,514 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:28,514 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:28,514 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:28,515 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:28,515 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:28,515 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:28,515 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/7b69e67612f14f53aaf7d1701cb93ac4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b9569645a2fb4b63bcfceca5e4799645, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/31ca373cbd5c43c391aed6a59a447f26, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a287c903bebd4c79b425170a7b641049] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=48.9 K 2024-11-13T18:14:28,515 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b69e67612f14f53aaf7d1701cb93ac4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731521665486 2024-11-13T18:14:28,516 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9569645a2fb4b63bcfceca5e4799645, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731521665723 2024-11-13T18:14:28,516 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31ca373cbd5c43c391aed6a59a447f26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731521666450 2024-11-13T18:14:28,516 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a287c903bebd4c79b425170a7b641049, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731521667937 2024-11-13T18:14:28,531 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:28,532 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/67849dcfb43f497c8ad86de342fd436c is 50, key is test_row_0/C:col10/1731521667937/Put/seqid=0 2024-11-13T18:14:28,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742444_1620 (size=13323) 2024-11-13T18:14:28,566 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/67849dcfb43f497c8ad86de342fd436c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/67849dcfb43f497c8ad86de342fd436c 2024-11-13T18:14:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:28,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:14:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:28,575 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into 67849dcfb43f497c8ad86de342fd436c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:28,575 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:28,575 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=12, startTime=1731521668371; duration=0sec 2024-11-13T18:14:28,575 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:28,575 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:28,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/df67241fdeb340a5914cc5d32ec478bf is 50, key is test_row_0/A:col10/1731521668188/Put/seqid=0 2024-11-13T18:14:28,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742445_1621 (size=14741) 2024-11-13T18:14:28,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/df67241fdeb340a5914cc5d32ec478bf 2024-11-13T18:14:28,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/12fb24fd4dcc45f3822bf476fa78505c is 50, key is test_row_0/B:col10/1731521668188/Put/seqid=0 2024-11-13T18:14:28,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742446_1622 (size=12301) 2024-11-13T18:14:28,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/12fb24fd4dcc45f3822bf476fa78505c 2024-11-13T18:14:28,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/d368947c0f694f06815a5392fba8700d is 50, key is test_row_0/C:col10/1731521668188/Put/seqid=0 2024-11-13T18:14:28,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742447_1623 (size=12301) 2024-11-13T18:14:28,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/d368947c0f694f06815a5392fba8700d 2024-11-13T18:14:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/df67241fdeb340a5914cc5d32ec478bf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/df67241fdeb340a5914cc5d32ec478bf 2024-11-13T18:14:28,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/df67241fdeb340a5914cc5d32ec478bf, entries=200, sequenceid=527, filesize=14.4 K 2024-11-13T18:14:28,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/12fb24fd4dcc45f3822bf476fa78505c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/12fb24fd4dcc45f3822bf476fa78505c 2024-11-13T18:14:28,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/12fb24fd4dcc45f3822bf476fa78505c, entries=150, sequenceid=527, filesize=12.0 K 2024-11-13T18:14:28,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/d368947c0f694f06815a5392fba8700d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/d368947c0f694f06815a5392fba8700d 2024-11-13T18:14:28,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/d368947c0f694f06815a5392fba8700d, entries=150, sequenceid=527, filesize=12.0 K 2024-11-13T18:14:28,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=107.34 KB/109920 for 278d3a3ebd6be30bccd59b5fe89f892a in 114ms, sequenceid=527, compaction requested=false 2024-11-13T18:14:28,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:28,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:14:28,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:28,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:28,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:28,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:28,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:28,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:28,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/a883fe7e82d44f61bff2fa441134253c is 50, key is test_row_0/A:col10/1731521668668/Put/seqid=0 2024-11-13T18:14:28,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742448_1624 (size=14741) 2024-11-13T18:14:28,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/a883fe7e82d44f61bff2fa441134253c 2024-11-13T18:14:28,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/1757892a0faf4856ab64dedd035b41f1 is 50, key is test_row_0/B:col10/1731521668668/Put/seqid=0 2024-11-13T18:14:28,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742449_1625 (size=12301) 2024-11-13T18:14:28,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521728743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521728748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521728864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521728873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37492 deadline: 1731521728905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,915 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 278d3a3ebd6be30bccd59b5fe89f892a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,919 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/c9362673870c4a65a1cb34b2462a6e07 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c9362673870c4a65a1cb34b2462a6e07 2024-11-13T18:14:28,924 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into c9362673870c4a65a1cb34b2462a6e07(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:28,924 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:28,924 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=12, startTime=1731521668370; duration=0sec 2024-11-13T18:14:28,924 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:28,924 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:28,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731521728949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,965 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in 278d3a3ebd6be30bccd59b5fe89f892a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8307 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37506 deadline: 1731521728956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:28,975 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_2' in 278d3a3ebd6be30bccd59b5fe89f892a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8316 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:28,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:28,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:14:28,989 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:28,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-13T18:14:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:14:29,000 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:29,001 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:29,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:29,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521729084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:14:29,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521729095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:29,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/1757892a0faf4856ab64dedd035b41f1 2024-11-13T18:14:29,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f1884dff55854e8eb4980218f371d6ac is 50, key is test_row_0/C:col10/1731521668668/Put/seqid=0 2024-11-13T18:14:29,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T18:14:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:29,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:29,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742450_1626 (size=12301) 2024-11-13T18:14:29,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f1884dff55854e8eb4980218f371d6ac 2024-11-13T18:14:29,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/a883fe7e82d44f61bff2fa441134253c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/a883fe7e82d44f61bff2fa441134253c 2024-11-13T18:14:29,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/a883fe7e82d44f61bff2fa441134253c, entries=200, sequenceid=548, filesize=14.4 K 2024-11-13T18:14:29,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/1757892a0faf4856ab64dedd035b41f1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1757892a0faf4856ab64dedd035b41f1 2024-11-13T18:14:29,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1757892a0faf4856ab64dedd035b41f1, entries=150, sequenceid=548, filesize=12.0 K 2024-11-13T18:14:29,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f1884dff55854e8eb4980218f371d6ac as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f1884dff55854e8eb4980218f371d6ac 2024-11-13T18:14:29,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f1884dff55854e8eb4980218f371d6ac, entries=150, sequenceid=548, filesize=12.0 K 2024-11-13T18:14:29,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 278d3a3ebd6be30bccd59b5fe89f892a in 514ms, sequenceid=548, compaction requested=true 2024-11-13T18:14:29,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:29,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:29,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:29,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:29,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:29,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:29,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:14:29,201 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:29,201 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:29,202 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:29,202 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:29,202 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:29,203 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/67849dcfb43f497c8ad86de342fd436c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/d368947c0f694f06815a5392fba8700d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f1884dff55854e8eb4980218f371d6ac] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=37.0 K 2024-11-13T18:14:29,203 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:29,203 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:29,203 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:29,203 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d23125d92c7b4e1c9643295d185052c6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/df67241fdeb340a5914cc5d32ec478bf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/a883fe7e82d44f61bff2fa441134253c] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=41.8 K 2024-11-13T18:14:29,203 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d23125d92c7b4e1c9643295d185052c6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731521667937 2024-11-13T18:14:29,203 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 67849dcfb43f497c8ad86de342fd436c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731521667937 2024-11-13T18:14:29,204 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d368947c0f694f06815a5392fba8700d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731521668158 2024-11-13T18:14:29,204 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting df67241fdeb340a5914cc5d32ec478bf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731521668158 2024-11-13T18:14:29,204 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a883fe7e82d44f61bff2fa441134253c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731521668654 2024-11-13T18:14:29,204 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f1884dff55854e8eb4980218f371d6ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731521668668 2024-11-13T18:14:29,223 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:29,224 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/c0f6b9f1581c42049870440daede4e33 is 50, key is test_row_0/A:col10/1731521668668/Put/seqid=0 2024-11-13T18:14:29,229 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:29,229 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f493944af174436cb87498a1f5ef18d9 is 50, key is test_row_0/C:col10/1731521668668/Put/seqid=0 2024-11-13T18:14:29,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742451_1627 (size=13425) 2024-11-13T18:14:29,274 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/c0f6b9f1581c42049870440daede4e33 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c0f6b9f1581c42049870440daede4e33 2024-11-13T18:14:29,281 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into c0f6b9f1581c42049870440daede4e33(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:29,281 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:29,281 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=13, startTime=1731521669201; duration=0sec 2024-11-13T18:14:29,281 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:29,281 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:29,282 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:29,282 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:29,283 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:29,283 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:29,283 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c9362673870c4a65a1cb34b2462a6e07, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/12fb24fd4dcc45f3822bf476fa78505c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1757892a0faf4856ab64dedd035b41f1] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=37.0 K 2024-11-13T18:14:29,283 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9362673870c4a65a1cb34b2462a6e07, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731521667937 2024-11-13T18:14:29,283 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12fb24fd4dcc45f3822bf476fa78505c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731521668158 2024-11-13T18:14:29,284 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1757892a0faf4856ab64dedd035b41f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731521668668 2024-11-13T18:14:29,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742452_1628 (size=13425) 2024-11-13T18:14:29,294 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#538 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:29,295 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/16c9e5492afe41219599531ce6445a77 is 50, key is test_row_0/B:col10/1731521668668/Put/seqid=0 2024-11-13T18:14:29,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T18:14:29,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:29,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:14:29,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:29,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:29,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:29,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:29,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:29,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:14:29,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742453_1629 (size=13425) 2024-11-13T18:14:29,323 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/16c9e5492afe41219599531ce6445a77 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/16c9e5492afe41219599531ce6445a77 2024-11-13T18:14:29,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/b4ffcbb0601e47f7ac0b080e08974eb3 is 50, key is test_row_0/A:col10/1731521668741/Put/seqid=0 2024-11-13T18:14:29,331 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into 16c9e5492afe41219599531ce6445a77(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:29,331 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:29,331 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=13, startTime=1731521669201; duration=0sec 2024-11-13T18:14:29,331 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:29,331 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:29,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742454_1630 (size=12301) 2024-11-13T18:14:29,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:29,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521729507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:29,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521729516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:14:29,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521729627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:29,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521729634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:29,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:29,705 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f493944af174436cb87498a1f5ef18d9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f493944af174436cb87498a1f5ef18d9 2024-11-13T18:14:29,710 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into f493944af174436cb87498a1f5ef18d9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:29,710 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:29,710 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=13, startTime=1731521669201; duration=0sec 2024-11-13T18:14:29,710 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:29,710 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:29,741 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/b4ffcbb0601e47f7ac0b080e08974eb3 2024-11-13T18:14:29,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/91e800e1220541b5beb05c8774a9a665 is 50, key is test_row_0/B:col10/1731521668741/Put/seqid=0 2024-11-13T18:14:29,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742455_1631 (size=12301) 2024-11-13T18:14:29,833 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/91e800e1220541b5beb05c8774a9a665 2024-11-13T18:14:29,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521729834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:29,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/aa8c00ff68944b3f887179c492368e0c is 50, key is test_row_0/C:col10/1731521668741/Put/seqid=0 2024-11-13T18:14:29,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521729854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:29,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742456_1632 (size=12301) 2024-11-13T18:14:29,883 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/aa8c00ff68944b3f887179c492368e0c 2024-11-13T18:14:29,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/b4ffcbb0601e47f7ac0b080e08974eb3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b4ffcbb0601e47f7ac0b080e08974eb3 2024-11-13T18:14:29,893 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b4ffcbb0601e47f7ac0b080e08974eb3, entries=150, sequenceid=562, filesize=12.0 K 2024-11-13T18:14:29,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/91e800e1220541b5beb05c8774a9a665 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/91e800e1220541b5beb05c8774a9a665 2024-11-13T18:14:29,898 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/91e800e1220541b5beb05c8774a9a665, entries=150, sequenceid=562, filesize=12.0 K 2024-11-13T18:14:29,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/aa8c00ff68944b3f887179c492368e0c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/aa8c00ff68944b3f887179c492368e0c 2024-11-13T18:14:29,919 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/aa8c00ff68944b3f887179c492368e0c, entries=150, sequenceid=562, filesize=12.0 K 2024-11-13T18:14:29,920 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 278d3a3ebd6be30bccd59b5fe89f892a in 607ms, sequenceid=562, compaction requested=false 2024-11-13T18:14:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-13T18:14:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-13T18:14:29,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-13T18:14:29,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 919 msec 2024-11-13T18:14:29,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 936 msec 2024-11-13T18:14:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:14:30,134 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:30,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-13T18:14:30,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:14:30,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:30,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:30,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T18:14:30,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:30,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:30,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:30,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:30,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:30,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:30,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/e2ebcc98fb324bcdbc6719c64b990c3e is 50, key is test_row_0/A:col10/1731521670157/Put/seqid=0 2024-11-13T18:14:30,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521730211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:30,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742457_1633 (size=14741) 2024-11-13T18:14:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:30,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521730216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:14:30,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:14:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:30,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521730333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:30,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521730344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:30,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:14:30,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:30,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:14:30,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521730544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:30,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521730563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:30,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:14:30,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:30,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:30,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:30,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/e2ebcc98fb324bcdbc6719c64b990c3e 2024-11-13T18:14:30,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/2af106b1da3d4ee4a6b4792b32ec7984 is 50, key is test_row_0/B:col10/1731521670157/Put/seqid=0 2024-11-13T18:14:30,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742458_1634 (size=12301) 2024-11-13T18:14:30,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/2af106b1da3d4ee4a6b4792b32ec7984 2024-11-13T18:14:30,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/94dca408274b4c89b2b8a3501ca7e9b4 is 50, key is test_row_0/C:col10/1731521670157/Put/seqid=0 2024-11-13T18:14:30,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742459_1635 (size=12301) 2024-11-13T18:14:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:14:30,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:14:30,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:30,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:30,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:30,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521730864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:30,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521730885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:30,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:14:30,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:30,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:30,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:30,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:31,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:14:31,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:31,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:31,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:31,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:31,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:31,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/94dca408274b4c89b2b8a3501ca7e9b4 2024-11-13T18:14:31,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/e2ebcc98fb324bcdbc6719c64b990c3e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e2ebcc98fb324bcdbc6719c64b990c3e 2024-11-13T18:14:31,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e2ebcc98fb324bcdbc6719c64b990c3e, entries=200, sequenceid=585, filesize=14.4 K 2024-11-13T18:14:31,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/2af106b1da3d4ee4a6b4792b32ec7984 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/2af106b1da3d4ee4a6b4792b32ec7984 2024-11-13T18:14:31,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/2af106b1da3d4ee4a6b4792b32ec7984, entries=150, sequenceid=585, filesize=12.0 K 2024-11-13T18:14:31,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/94dca408274b4c89b2b8a3501ca7e9b4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/94dca408274b4c89b2b8a3501ca7e9b4 2024-11-13T18:14:31,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/94dca408274b4c89b2b8a3501ca7e9b4, entries=150, sequenceid=585, filesize=12.0 K 2024-11-13T18:14:31,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 278d3a3ebd6be30bccd59b5fe89f892a in 1067ms, sequenceid=585, compaction requested=true 2024-11-13T18:14:31,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:31,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:31,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:31,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:31,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:31,226 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:31,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:31,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:31,226 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:31,227 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:31,227 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:31,227 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:31,227 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c0f6b9f1581c42049870440daede4e33, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b4ffcbb0601e47f7ac0b080e08974eb3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e2ebcc98fb324bcdbc6719c64b990c3e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=39.5 K 2024-11-13T18:14:31,227 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:31,227 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:31,227 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:31,227 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/16c9e5492afe41219599531ce6445a77, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/91e800e1220541b5beb05c8774a9a665, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/2af106b1da3d4ee4a6b4792b32ec7984] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=37.1 K 2024-11-13T18:14:31,227 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f6b9f1581c42049870440daede4e33, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731521668668 2024-11-13T18:14:31,227 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16c9e5492afe41219599531ce6445a77, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731521668668 2024-11-13T18:14:31,228 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91e800e1220541b5beb05c8774a9a665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731521668740 2024-11-13T18:14:31,228 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ffcbb0601e47f7ac0b080e08974eb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731521668740 2024-11-13T18:14:31,228 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2af106b1da3d4ee4a6b4792b32ec7984, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731521669506 2024-11-13T18:14:31,231 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e2ebcc98fb324bcdbc6719c64b990c3e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731521669491 2024-11-13T18:14:31,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:14:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:31,239 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:14:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:31,247 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#545 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:31,247 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/53c246ad29194f70b2d8bfb2a1c1ef48 is 50, key is test_row_0/B:col10/1731521670157/Put/seqid=0 2024-11-13T18:14:31,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/f9e05a81120a4396bf2cfb962f76f0cd is 50, key is test_row_0/A:col10/1731521670194/Put/seqid=0 2024-11-13T18:14:31,262 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#547 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:31,262 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/16b96be3596f44f49204dd84bbb5a65d is 50, key is test_row_0/A:col10/1731521670157/Put/seqid=0 2024-11-13T18:14:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:14:31,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742462_1638 (size=13527) 2024-11-13T18:14:31,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742460_1636 (size=13527) 2024-11-13T18:14:31,332 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/53c246ad29194f70b2d8bfb2a1c1ef48 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/53c246ad29194f70b2d8bfb2a1c1ef48 2024-11-13T18:14:31,340 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into 53c246ad29194f70b2d8bfb2a1c1ef48(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:31,340 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:31,341 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=13, startTime=1731521671225; duration=0sec 2024-11-13T18:14:31,341 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:31,341 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:31,341 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:31,342 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:31,342 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:31,342 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:31,342 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f493944af174436cb87498a1f5ef18d9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/aa8c00ff68944b3f887179c492368e0c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/94dca408274b4c89b2b8a3501ca7e9b4] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=37.1 K 2024-11-13T18:14:31,343 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f493944af174436cb87498a1f5ef18d9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731521668668 2024-11-13T18:14:31,343 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa8c00ff68944b3f887179c492368e0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731521668740 2024-11-13T18:14:31,344 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94dca408274b4c89b2b8a3501ca7e9b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731521669506 2024-11-13T18:14:31,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742461_1637 (size=12301) 2024-11-13T18:14:31,363 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#548 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:31,364 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f45a9f260af141e99f95da312ba01022 is 50, key is test_row_0/C:col10/1731521670157/Put/seqid=0 2024-11-13T18:14:31,364 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/f9e05a81120a4396bf2cfb962f76f0cd 2024-11-13T18:14:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:31,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:31,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/b75a57d2c2f04918973b709cd631b95b is 50, key is test_row_0/B:col10/1731521670194/Put/seqid=0 2024-11-13T18:14:31,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742463_1639 (size=13527) 2024-11-13T18:14:31,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742464_1640 (size=12301) 2024-11-13T18:14:31,456 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/b75a57d2c2f04918973b709cd631b95b 2024-11-13T18:14:31,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/689cf413d7af4c2b98f494932d597ee3 is 50, key is test_row_0/C:col10/1731521670194/Put/seqid=0 2024-11-13T18:14:31,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742465_1641 (size=12301) 2024-11-13T18:14:31,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/689cf413d7af4c2b98f494932d597ee3 2024-11-13T18:14:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/f9e05a81120a4396bf2cfb962f76f0cd as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f9e05a81120a4396bf2cfb962f76f0cd 2024-11-13T18:14:31,569 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f9e05a81120a4396bf2cfb962f76f0cd, entries=150, sequenceid=596, filesize=12.0 K 2024-11-13T18:14:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/b75a57d2c2f04918973b709cd631b95b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b75a57d2c2f04918973b709cd631b95b 2024-11-13T18:14:31,582 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b75a57d2c2f04918973b709cd631b95b, entries=150, sequenceid=596, filesize=12.0 K 2024-11-13T18:14:31,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/689cf413d7af4c2b98f494932d597ee3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/689cf413d7af4c2b98f494932d597ee3 2024-11-13T18:14:31,601 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/689cf413d7af4c2b98f494932d597ee3, entries=150, sequenceid=596, filesize=12.0 K 2024-11-13T18:14:31,602 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=93.93 KB/96180 for 278d3a3ebd6be30bccd59b5fe89f892a in 363ms, sequenceid=596, compaction requested=false 2024-11-13T18:14:31,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:31,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:31,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-13T18:14:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-13T18:14:31,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-13T18:14:31,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4710 sec 2024-11-13T18:14:31,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4820 sec 2024-11-13T18:14:31,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:14:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:31,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:31,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:31,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:31,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:31,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:31,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:31,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/e01eb6cf4c99430cbfecf9cfa796e194 is 50, key is test_row_0/A:col10/1731521671614/Put/seqid=0 2024-11-13T18:14:31,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742466_1642 (size=12301) 2024-11-13T18:14:31,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/e01eb6cf4c99430cbfecf9cfa796e194 2024-11-13T18:14:31,725 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/16b96be3596f44f49204dd84bbb5a65d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/16b96be3596f44f49204dd84bbb5a65d 2024-11-13T18:14:31,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/d90a88fe27954a799ccb1bcb9196a785 is 50, key is test_row_0/B:col10/1731521671614/Put/seqid=0 2024-11-13T18:14:31,745 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into 16b96be3596f44f49204dd84bbb5a65d(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:31,745 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:31,745 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=13, startTime=1731521671225; duration=0sec 2024-11-13T18:14:31,745 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:31,745 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:31,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521731762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:31,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742467_1643 (size=12301) 2024-11-13T18:14:31,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521731774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:31,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/d90a88fe27954a799ccb1bcb9196a785 2024-11-13T18:14:31,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/0729d2fb3b4e45fd965eb31c832b138d is 50, key is test_row_0/C:col10/1731521671614/Put/seqid=0 2024-11-13T18:14:31,836 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/f45a9f260af141e99f95da312ba01022 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f45a9f260af141e99f95da312ba01022 2024-11-13T18:14:31,865 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into f45a9f260af141e99f95da312ba01022(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:31,865 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:31,865 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=13, startTime=1731521671226; duration=0sec 2024-11-13T18:14:31,865 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:31,865 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:31,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742468_1644 (size=12301) 2024-11-13T18:14:31,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521731884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:31,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:31,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:31,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/0729d2fb3b4e45fd965eb31c832b138d 2024-11-13T18:14:31,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521731904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:31,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/e01eb6cf4c99430cbfecf9cfa796e194 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e01eb6cf4c99430cbfecf9cfa796e194 2024-11-13T18:14:31,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e01eb6cf4c99430cbfecf9cfa796e194, entries=150, sequenceid=616, filesize=12.0 K 2024-11-13T18:14:31,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/d90a88fe27954a799ccb1bcb9196a785 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d90a88fe27954a799ccb1bcb9196a785 2024-11-13T18:14:32,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d90a88fe27954a799ccb1bcb9196a785, entries=150, sequenceid=616, filesize=12.0 K 2024-11-13T18:14:32,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/0729d2fb3b4e45fd965eb31c832b138d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0729d2fb3b4e45fd965eb31c832b138d 2024-11-13T18:14:32,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0729d2fb3b4e45fd965eb31c832b138d, entries=150, sequenceid=616, filesize=12.0 K 2024-11-13T18:14:32,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 278d3a3ebd6be30bccd59b5fe89f892a in 407ms, sequenceid=616, compaction requested=true 2024-11-13T18:14:32,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:32,029 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:32,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:32,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:32,029 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:32,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:32,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:32,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:32,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:32,033 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:32,034 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:32,034 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:32,034 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/16b96be3596f44f49204dd84bbb5a65d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f9e05a81120a4396bf2cfb962f76f0cd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e01eb6cf4c99430cbfecf9cfa796e194] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=37.2 K 2024-11-13T18:14:32,038 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:32,038 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:32,038 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:32,039 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/53c246ad29194f70b2d8bfb2a1c1ef48, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b75a57d2c2f04918973b709cd631b95b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d90a88fe27954a799ccb1bcb9196a785] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=37.2 K 2024-11-13T18:14:32,039 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16b96be3596f44f49204dd84bbb5a65d, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731521669506 2024-11-13T18:14:32,042 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c246ad29194f70b2d8bfb2a1c1ef48, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731521669506 2024-11-13T18:14:32,042 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9e05a81120a4396bf2cfb962f76f0cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1731521670194 2024-11-13T18:14:32,042 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b75a57d2c2f04918973b709cd631b95b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1731521670194 2024-11-13T18:14:32,043 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e01eb6cf4c99430cbfecf9cfa796e194, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731521671585 2024-11-13T18:14:32,043 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d90a88fe27954a799ccb1bcb9196a785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731521671585 2024-11-13T18:14:32,063 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#554 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:32,064 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/ac5edbad0eec454892137e4d17a1e25c is 50, key is test_row_0/A:col10/1731521671614/Put/seqid=0 2024-11-13T18:14:32,068 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#555 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:32,068 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/d1e56c0f06444bc1bb7e223f48e43256 is 50, key is test_row_0/B:col10/1731521671614/Put/seqid=0 2024-11-13T18:14:32,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742469_1645 (size=13629) 2024-11-13T18:14:32,109 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/ac5edbad0eec454892137e4d17a1e25c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ac5edbad0eec454892137e4d17a1e25c 2024-11-13T18:14:32,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742470_1646 (size=13629) 2024-11-13T18:14:32,121 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into ac5edbad0eec454892137e4d17a1e25c(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:32,121 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:32,121 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=13, startTime=1731521672028; duration=0sec 2024-11-13T18:14:32,121 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:32,121 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:32,122 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:32,125 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:32,125 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:32,126 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:32,126 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f45a9f260af141e99f95da312ba01022, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/689cf413d7af4c2b98f494932d597ee3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0729d2fb3b4e45fd965eb31c832b138d] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=37.2 K 2024-11-13T18:14:32,126 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f45a9f260af141e99f95da312ba01022, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731521669506 2024-11-13T18:14:32,127 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/d1e56c0f06444bc1bb7e223f48e43256 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d1e56c0f06444bc1bb7e223f48e43256 2024-11-13T18:14:32,127 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 689cf413d7af4c2b98f494932d597ee3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1731521670194 2024-11-13T18:14:32,128 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0729d2fb3b4e45fd965eb31c832b138d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731521671585 2024-11-13T18:14:32,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T18:14:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:32,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/b945d67651a54c9ebe25c246ff677277 is 50, key is test_row_0/A:col10/1731521672128/Put/seqid=0 2024-11-13T18:14:32,138 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into d1e56c0f06444bc1bb7e223f48e43256(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:32,138 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:32,138 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=13, startTime=1731521672029; duration=0sec 2024-11-13T18:14:32,138 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:32,138 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:32,151 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#557 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:32,152 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/a851d59387e543eb813d737b35e61321 is 50, key is test_row_0/C:col10/1731521671614/Put/seqid=0 2024-11-13T18:14:32,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742471_1647 (size=14741) 2024-11-13T18:14:32,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742472_1648 (size=13629) 2024-11-13T18:14:32,213 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/a851d59387e543eb813d737b35e61321 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a851d59387e543eb813d737b35e61321 2024-11-13T18:14:32,250 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into a851d59387e543eb813d737b35e61321(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:32,250 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:32,251 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=13, startTime=1731521672029; duration=0sec 2024-11-13T18:14:32,251 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:32,251 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:32,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521732278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:14:32,294 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:32,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-13T18:14:32,297 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:14:32,297 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:32,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:32,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521732284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:32,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521732394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:14:32,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521732414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:32,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T18:14:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:32,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:32,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/b945d67651a54c9ebe25c246ff677277 2024-11-13T18:14:32,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T18:14:32,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:32,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:32,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:32,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:32,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:14:32,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521732614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:32,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521732634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:32,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/23d483aca961452c8fc11748878d310f is 50, key is test_row_0/B:col10/1731521672128/Put/seqid=0 2024-11-13T18:14:32,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742473_1649 (size=12301) 2024-11-13T18:14:32,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/23d483aca961452c8fc11748878d310f 2024-11-13T18:14:32,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/0cdd3c89c654407aa698cf63eb9c358e is 50, key is test_row_0/C:col10/1731521672128/Put/seqid=0 2024-11-13T18:14:32,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742474_1650 (size=12301) 2024-11-13T18:14:32,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/0cdd3c89c654407aa698cf63eb9c358e 2024-11-13T18:14:32,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/b945d67651a54c9ebe25c246ff677277 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b945d67651a54c9ebe25c246ff677277 2024-11-13T18:14:32,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b945d67651a54c9ebe25c246ff677277, entries=200, sequenceid=633, filesize=14.4 K 2024-11-13T18:14:32,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/23d483aca961452c8fc11748878d310f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/23d483aca961452c8fc11748878d310f 2024-11-13T18:14:32,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/23d483aca961452c8fc11748878d310f, entries=150, sequenceid=633, filesize=12.0 K 2024-11-13T18:14:32,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/0cdd3c89c654407aa698cf63eb9c358e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0cdd3c89c654407aa698cf63eb9c358e 2024-11-13T18:14:32,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T18:14:32,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:32,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:32,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:32,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:32,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:32,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0cdd3c89c654407aa698cf63eb9c358e, entries=150, sequenceid=633, filesize=12.0 K 2024-11-13T18:14:32,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 278d3a3ebd6be30bccd59b5fe89f892a in 637ms, sequenceid=633, compaction requested=false 2024-11-13T18:14:32,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:32,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T18:14:32,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:32,924 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:14:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:14:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/894b3218fb7149b7bc985b7ae5d4f6f3 is 50, key is test_row_0/A:col10/1731521672184/Put/seqid=0 2024-11-13T18:14:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:32,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. as already flushing 2024-11-13T18:14:32,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742475_1651 (size=12301) 2024-11-13T18:14:33,057 INFO [Thread-2347 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2347. 2024-11-13T18:14:33,057 DEBUG [Thread-2347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:33,057 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:33,057 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:33,057 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:33,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521733043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:33,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521733045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:33,062 INFO [Thread-2345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2345. 2024-11-13T18:14:33,062 DEBUG [Thread-2345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:33,062 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:33,062 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:33,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:33,071 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-11-13T18:14:33,071 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:33,071 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:33,071 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:33,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:33,072 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-11-13T18:14:33,072 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:33,072 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:33,072 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:33,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:33,077 INFO [Thread-2343 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2343. 2024-11-13T18:14:33,077 DEBUG [Thread-2343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:33,077 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:33,077 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:33,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:33,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:33,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521733164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521733164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:33,347 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/894b3218fb7149b7bc985b7ae5d4f6f3 2024-11-13T18:14:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/59c2e83834b24d3e86527ed1d72637cc is 50, key is test_row_0/B:col10/1731521672184/Put/seqid=0 2024-11-13T18:14:33,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742476_1652 (size=12301) 2024-11-13T18:14:33,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521733374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:33,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:33,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521733377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:33,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:14:33,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:33,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37558 deadline: 1731521733684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37478 deadline: 1731521733684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 , the old value is region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., hostname=fda96cf025dc,44483,1731521526892, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:14:33,756 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/59c2e83834b24d3e86527ed1d72637cc 2024-11-13T18:14:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/cc9a625802dc4c62a829b85fca45656e is 50, key is test_row_0/C:col10/1731521672184/Put/seqid=0 2024-11-13T18:14:33,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742477_1653 (size=12301) 2024-11-13T18:14:34,165 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/cc9a625802dc4c62a829b85fca45656e 2024-11-13T18:14:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/894b3218fb7149b7bc985b7ae5d4f6f3 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/894b3218fb7149b7bc985b7ae5d4f6f3 2024-11-13T18:14:34,171 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/894b3218fb7149b7bc985b7ae5d4f6f3, entries=150, sequenceid=653, filesize=12.0 K 2024-11-13T18:14:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/59c2e83834b24d3e86527ed1d72637cc as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/59c2e83834b24d3e86527ed1d72637cc 2024-11-13T18:14:34,175 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/59c2e83834b24d3e86527ed1d72637cc, entries=150, sequenceid=653, filesize=12.0 K 2024-11-13T18:14:34,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/cc9a625802dc4c62a829b85fca45656e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/cc9a625802dc4c62a829b85fca45656e 2024-11-13T18:14:34,178 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/cc9a625802dc4c62a829b85fca45656e, entries=150, sequenceid=653, filesize=12.0 K 2024-11-13T18:14:34,178 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 278d3a3ebd6be30bccd59b5fe89f892a in 1254ms, sequenceid=653, compaction requested=true 2024-11-13T18:14:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-13T18:14:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-13T18:14:34,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-13T18:14:34,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-13T18:14:34,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.8870 sec 2024-11-13T18:14:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:34,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:14:34,196 INFO [Thread-2330 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2330. 2024-11-13T18:14:34,196 DEBUG [Thread-2330 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:34,196 INFO [Thread-2336 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2336. 2024-11-13T18:14:34,196 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:34,196 DEBUG [Thread-2336 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:34,196 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:34,196 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:34,196 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:34,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:34,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:34,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7faaee0e166d4c0fbf5d4937095bad37 is 50, key is test_row_0/A:col10/1731521673041/Put/seqid=0 2024-11-13T18:14:34,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742478_1654 (size=12301) 2024-11-13T18:14:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:14:34,445 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:34,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7faaee0e166d4c0fbf5d4937095bad37 2024-11-13T18:14:34,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/5bea2d95be9a4849874bf6e2626cfc9e is 50, key is test_row_0/B:col10/1731521673041/Put/seqid=0 2024-11-13T18:14:34,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742479_1655 (size=12301) 2024-11-13T18:14:34,984 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:14:35,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/5bea2d95be9a4849874bf6e2626cfc9e 2024-11-13T18:14:35,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b8042a4f85d24d768f7fb43922c95ac9 is 50, key is test_row_0/C:col10/1731521673041/Put/seqid=0 2024-11-13T18:14:35,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742480_1656 (size=12301) 2024-11-13T18:14:35,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b8042a4f85d24d768f7fb43922c95ac9 2024-11-13T18:14:35,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/7faaee0e166d4c0fbf5d4937095bad37 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7faaee0e166d4c0fbf5d4937095bad37 2024-11-13T18:14:35,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7faaee0e166d4c0fbf5d4937095bad37, entries=150, sequenceid=669, filesize=12.0 K 2024-11-13T18:14:35,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/5bea2d95be9a4849874bf6e2626cfc9e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5bea2d95be9a4849874bf6e2626cfc9e 2024-11-13T18:14:35,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5bea2d95be9a4849874bf6e2626cfc9e, entries=150, sequenceid=669, filesize=12.0 K 2024-11-13T18:14:35,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/b8042a4f85d24d768f7fb43922c95ac9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b8042a4f85d24d768f7fb43922c95ac9 2024-11-13T18:14:35,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b8042a4f85d24d768f7fb43922c95ac9, entries=150, sequenceid=669, filesize=12.0 K 2024-11-13T18:14:35,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 278d3a3ebd6be30bccd59b5fe89f892a in 1245ms, sequenceid=669, compaction requested=true 2024-11-13T18:14:35,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:35,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:35,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:35,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:35,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:35,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 278d3a3ebd6be30bccd59b5fe89f892a:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:35,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:35,441 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:35,441 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:35,442 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:35,442 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:35,442 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/A is initiating minor compaction (all files) 2024-11-13T18:14:35,442 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/B is initiating minor compaction (all files) 2024-11-13T18:14:35,442 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/A in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:35,442 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/B in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:35,442 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ac5edbad0eec454892137e4d17a1e25c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b945d67651a54c9ebe25c246ff677277, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/894b3218fb7149b7bc985b7ae5d4f6f3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7faaee0e166d4c0fbf5d4937095bad37] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=51.7 K 2024-11-13T18:14:35,442 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d1e56c0f06444bc1bb7e223f48e43256, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/23d483aca961452c8fc11748878d310f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/59c2e83834b24d3e86527ed1d72637cc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5bea2d95be9a4849874bf6e2626cfc9e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=49.3 K 2024-11-13T18:14:35,442 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac5edbad0eec454892137e4d17a1e25c, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731521671585 2024-11-13T18:14:35,442 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e56c0f06444bc1bb7e223f48e43256, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731521671585 2024-11-13T18:14:35,443 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b945d67651a54c9ebe25c246ff677277, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=633, earliestPutTs=1731521671732 2024-11-13T18:14:35,443 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d483aca961452c8fc11748878d310f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=633, earliestPutTs=1731521671754 2024-11-13T18:14:35,443 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 894b3218fb7149b7bc985b7ae5d4f6f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1731521672184 2024-11-13T18:14:35,443 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c2e83834b24d3e86527ed1d72637cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1731521672184 2024-11-13T18:14:35,443 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7faaee0e166d4c0fbf5d4937095bad37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1731521673041 2024-11-13T18:14:35,443 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bea2d95be9a4849874bf6e2626cfc9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1731521673041 2024-11-13T18:14:35,452 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#B#compaction#566 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:35,452 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#A#compaction#567 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:35,452 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/dcf5f464d1ea42dc83d6d07c52718904 is 50, key is test_row_0/B:col10/1731521673041/Put/seqid=0 2024-11-13T18:14:35,452 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/c8f6e1123db54f19be3f3b562995d20d is 50, key is test_row_0/A:col10/1731521673041/Put/seqid=0 2024-11-13T18:14:35,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742481_1657 (size=13765) 2024-11-13T18:14:35,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742482_1658 (size=13765) 2024-11-13T18:14:35,864 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/dcf5f464d1ea42dc83d6d07c52718904 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/dcf5f464d1ea42dc83d6d07c52718904 2024-11-13T18:14:35,864 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/c8f6e1123db54f19be3f3b562995d20d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c8f6e1123db54f19be3f3b562995d20d 2024-11-13T18:14:35,868 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/A of 278d3a3ebd6be30bccd59b5fe89f892a into c8f6e1123db54f19be3f3b562995d20d(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:35,868 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/B of 278d3a3ebd6be30bccd59b5fe89f892a into dcf5f464d1ea42dc83d6d07c52718904(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:35,868 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:35,868 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:35,868 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/A, priority=12, startTime=1731521675441; duration=0sec 2024-11-13T18:14:35,868 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/B, priority=12, startTime=1731521675441; duration=0sec 2024-11-13T18:14:35,868 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:35,868 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:35,868 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:A 2024-11-13T18:14:35,868 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:B 2024-11-13T18:14:35,868 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:35,869 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:35,869 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): 278d3a3ebd6be30bccd59b5fe89f892a/C is initiating minor compaction (all files) 2024-11-13T18:14:35,869 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 278d3a3ebd6be30bccd59b5fe89f892a/C in TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:35,869 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a851d59387e543eb813d737b35e61321, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0cdd3c89c654407aa698cf63eb9c358e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/cc9a625802dc4c62a829b85fca45656e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b8042a4f85d24d768f7fb43922c95ac9] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp, totalSize=49.3 K 2024-11-13T18:14:35,869 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a851d59387e543eb813d737b35e61321, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731521671585 2024-11-13T18:14:35,869 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cdd3c89c654407aa698cf63eb9c358e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=633, earliestPutTs=1731521671754 2024-11-13T18:14:35,870 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc9a625802dc4c62a829b85fca45656e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1731521672184 2024-11-13T18:14:35,870 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8042a4f85d24d768f7fb43922c95ac9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1731521673041 2024-11-13T18:14:35,876 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 278d3a3ebd6be30bccd59b5fe89f892a#C#compaction#568 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:35,877 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/92f1b3f2b4f044aab4ccc5fc645ea34e is 50, key is test_row_0/C:col10/1731521673041/Put/seqid=0 2024-11-13T18:14:35,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742483_1659 (size=13765) 2024-11-13T18:14:36,285 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/92f1b3f2b4f044aab4ccc5fc645ea34e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/92f1b3f2b4f044aab4ccc5fc645ea34e 2024-11-13T18:14:36,289 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 278d3a3ebd6be30bccd59b5fe89f892a/C of 278d3a3ebd6be30bccd59b5fe89f892a into 92f1b3f2b4f044aab4ccc5fc645ea34e(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:36,289 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 278d3a3ebd6be30bccd59b5fe89f892a: 2024-11-13T18:14:36,289 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., storeName=278d3a3ebd6be30bccd59b5fe89f892a/C, priority=12, startTime=1731521675441; duration=0sec 2024-11-13T18:14:36,289 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:36,289 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 278d3a3ebd6be30bccd59b5fe89f892a:C 2024-11-13T18:14:38,956 INFO [Thread-2332 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2332. 2024-11-13T18:14:38,956 DEBUG [Thread-2332 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:38,956 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:38,956 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:38,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:38,995 INFO [Thread-2328 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2328. 2024-11-13T18:14:38,995 DEBUG [Thread-2328 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:38,995 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:38,995 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:38,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:39,015 INFO [Thread-2334 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2334. 2024-11-13T18:14:39,016 DEBUG [Thread-2334 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:14:39,016 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:39,016 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:39,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 195 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 202 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 796 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2388 rows 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 802 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2406 rows 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 800 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2400 rows 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 797 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2391 rows 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 797 2024-11-13T18:14:39,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2391 rows 2024-11-13T18:14:39,016 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:14:39,016 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:14:39,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:39,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:39,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:14:39,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:14:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T18:14:39,021 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521679020"}]},"ts":"1731521679020"} 2024-11-13T18:14:39,022 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:14:39,022 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:14:39,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:14:39,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=278d3a3ebd6be30bccd59b5fe89f892a, UNASSIGN}] 2024-11-13T18:14:39,024 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=278d3a3ebd6be30bccd59b5fe89f892a, UNASSIGN 2024-11-13T18:14:39,025 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=278d3a3ebd6be30bccd59b5fe89f892a, regionState=CLOSING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:39,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=278d3a3ebd6be30bccd59b5fe89f892a, UNASSIGN because future has completed 2024-11-13T18:14:39,026 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:14:39,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:14:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T18:14:39,178 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:39,178 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:14:39,179 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 278d3a3ebd6be30bccd59b5fe89f892a, disabling compactions & flushes 2024-11-13T18:14:39,179 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:39,179 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:39,179 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. after waiting 0 ms 2024-11-13T18:14:39,179 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:39,179 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 278d3a3ebd6be30bccd59b5fe89f892a 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-13T18:14:39,179 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=A 2024-11-13T18:14:39,179 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:39,179 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=B 2024-11-13T18:14:39,179 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:39,179 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 278d3a3ebd6be30bccd59b5fe89f892a, store=C 2024-11-13T18:14:39,179 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:39,182 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/4c07ed6eb1084ca9a8bfa0b03270a8d5 is 50, key is test_row_0/A:col10/1731521678994/Put/seqid=0 2024-11-13T18:14:39,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742484_1660 (size=9857) 2024-11-13T18:14:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T18:14:39,586 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/4c07ed6eb1084ca9a8bfa0b03270a8d5 2024-11-13T18:14:39,594 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/cd370ddf10a84197888bc8cca2363760 is 50, key is test_row_0/B:col10/1731521678994/Put/seqid=0 2024-11-13T18:14:39,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742485_1661 (size=9857) 2024-11-13T18:14:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T18:14:39,999 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/cd370ddf10a84197888bc8cca2363760 2024-11-13T18:14:40,006 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/454c4a5442ec4e25a076589060c212d8 is 50, key is test_row_0/C:col10/1731521678994/Put/seqid=0 2024-11-13T18:14:40,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742486_1662 (size=9857) 2024-11-13T18:14:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T18:14:40,410 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/454c4a5442ec4e25a076589060c212d8 2024-11-13T18:14:40,414 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/A/4c07ed6eb1084ca9a8bfa0b03270a8d5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/4c07ed6eb1084ca9a8bfa0b03270a8d5 2024-11-13T18:14:40,417 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/4c07ed6eb1084ca9a8bfa0b03270a8d5, entries=100, sequenceid=678, filesize=9.6 K 2024-11-13T18:14:40,418 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/B/cd370ddf10a84197888bc8cca2363760 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/cd370ddf10a84197888bc8cca2363760 2024-11-13T18:14:40,420 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/cd370ddf10a84197888bc8cca2363760, entries=100, sequenceid=678, filesize=9.6 K 2024-11-13T18:14:40,421 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/.tmp/C/454c4a5442ec4e25a076589060c212d8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/454c4a5442ec4e25a076589060c212d8 2024-11-13T18:14:40,424 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/454c4a5442ec4e25a076589060c212d8, entries=100, sequenceid=678, filesize=9.6 K 2024-11-13T18:14:40,424 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 278d3a3ebd6be30bccd59b5fe89f892a in 1245ms, sequenceid=678, compaction requested=false 2024-11-13T18:14:40,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/cdd6af2d5d464293b752fe80554b8bad, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826e22ce6d054215a09b9673ef2cd948, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/dc71c7f9c2a44447973bedb6cf28358a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7cae99703da0433d9fd4c3f36a960a57, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e5d511b684894f469959918a7f29f72f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ddb4da72b2ba4fd896a822819cf09f16, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f8dd39c0043743788cfbb7b7410c2510, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/36084d7c87cc436796c4c6aea712b79f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1980099dbcc04ee18a54fb3414d24361, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6c629a8424354c199d4747d304e1e8c1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/37424b7b647f4059bc824bb45e211eff, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/2151ceb4b3334c568db2660ffc502b69, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7e5b2bdd6f724bdb8898b4215eb540b1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d3236b745d2b40f8a0bafb53e9c091a0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c21edc29690b440c97362456cebaa5e7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/fcb59e6983f34462a9bb1fe12af53bac, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826c3a7e0f14474694b0ea055376ccf2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7d30dcc3000b4c419a625340f64afaf8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/4499eb2bd5214dd9b8dd190a94f1e330, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/853c7bb3bcb24adbb48b03f24bd0be24, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6fd044c714f24fdc983816c0b2d883c3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1b169efc7f33465f83c8cc23d5290092, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c2ce5ae6c37f467aa13a6999c7c2bc20, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7ca628d5027847699bd6e4880b1b64a6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f032b0483b2445f98f2e1daf23c73c50, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/9e069843ba374797929b1d84065a4de1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/41241cc2395148848a6a8a8cc5e4a0ac, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/65779f7185c7442290d515b6c926e5ec, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/301723cab32b4ac7bc39553fdba223d4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/46993f4863c440f2abc2b85b405a0c23, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d23125d92c7b4e1c9643295d185052c6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/df67241fdeb340a5914cc5d32ec478bf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/a883fe7e82d44f61bff2fa441134253c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c0f6b9f1581c42049870440daede4e33, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b4ffcbb0601e47f7ac0b080e08974eb3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e2ebcc98fb324bcdbc6719c64b990c3e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/16b96be3596f44f49204dd84bbb5a65d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f9e05a81120a4396bf2cfb962f76f0cd, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ac5edbad0eec454892137e4d17a1e25c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e01eb6cf4c99430cbfecf9cfa796e194, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b945d67651a54c9ebe25c246ff677277, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/894b3218fb7149b7bc985b7ae5d4f6f3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7faaee0e166d4c0fbf5d4937095bad37] to archive 2024-11-13T18:14:40,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:14:40,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/cdd6af2d5d464293b752fe80554b8bad to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/cdd6af2d5d464293b752fe80554b8bad 2024-11-13T18:14:40,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826e22ce6d054215a09b9673ef2cd948 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826e22ce6d054215a09b9673ef2cd948 2024-11-13T18:14:40,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/dc71c7f9c2a44447973bedb6cf28358a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/dc71c7f9c2a44447973bedb6cf28358a 2024-11-13T18:14:40,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7cae99703da0433d9fd4c3f36a960a57 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7cae99703da0433d9fd4c3f36a960a57 2024-11-13T18:14:40,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e5d511b684894f469959918a7f29f72f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e5d511b684894f469959918a7f29f72f 2024-11-13T18:14:40,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ddb4da72b2ba4fd896a822819cf09f16 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ddb4da72b2ba4fd896a822819cf09f16 2024-11-13T18:14:40,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f8dd39c0043743788cfbb7b7410c2510 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f8dd39c0043743788cfbb7b7410c2510 2024-11-13T18:14:40,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/36084d7c87cc436796c4c6aea712b79f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/36084d7c87cc436796c4c6aea712b79f 2024-11-13T18:14:40,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1980099dbcc04ee18a54fb3414d24361 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1980099dbcc04ee18a54fb3414d24361 2024-11-13T18:14:40,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6c629a8424354c199d4747d304e1e8c1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6c629a8424354c199d4747d304e1e8c1 2024-11-13T18:14:40,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/37424b7b647f4059bc824bb45e211eff to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/37424b7b647f4059bc824bb45e211eff 2024-11-13T18:14:40,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/2151ceb4b3334c568db2660ffc502b69 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/2151ceb4b3334c568db2660ffc502b69 2024-11-13T18:14:40,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7e5b2bdd6f724bdb8898b4215eb540b1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7e5b2bdd6f724bdb8898b4215eb540b1 2024-11-13T18:14:40,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d3236b745d2b40f8a0bafb53e9c091a0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d3236b745d2b40f8a0bafb53e9c091a0 2024-11-13T18:14:40,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c21edc29690b440c97362456cebaa5e7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c21edc29690b440c97362456cebaa5e7 2024-11-13T18:14:40,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/fcb59e6983f34462a9bb1fe12af53bac to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/fcb59e6983f34462a9bb1fe12af53bac 2024-11-13T18:14:40,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826c3a7e0f14474694b0ea055376ccf2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/826c3a7e0f14474694b0ea055376ccf2 2024-11-13T18:14:40,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7d30dcc3000b4c419a625340f64afaf8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7d30dcc3000b4c419a625340f64afaf8 2024-11-13T18:14:40,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/4499eb2bd5214dd9b8dd190a94f1e330 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/4499eb2bd5214dd9b8dd190a94f1e330 2024-11-13T18:14:40,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/853c7bb3bcb24adbb48b03f24bd0be24 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/853c7bb3bcb24adbb48b03f24bd0be24 2024-11-13T18:14:40,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6fd044c714f24fdc983816c0b2d883c3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/6fd044c714f24fdc983816c0b2d883c3 2024-11-13T18:14:40,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1b169efc7f33465f83c8cc23d5290092 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/1b169efc7f33465f83c8cc23d5290092 2024-11-13T18:14:40,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c2ce5ae6c37f467aa13a6999c7c2bc20 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c2ce5ae6c37f467aa13a6999c7c2bc20 2024-11-13T18:14:40,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7ca628d5027847699bd6e4880b1b64a6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7ca628d5027847699bd6e4880b1b64a6 2024-11-13T18:14:40,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f032b0483b2445f98f2e1daf23c73c50 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f032b0483b2445f98f2e1daf23c73c50 2024-11-13T18:14:40,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/9e069843ba374797929b1d84065a4de1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/9e069843ba374797929b1d84065a4de1 2024-11-13T18:14:40,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/41241cc2395148848a6a8a8cc5e4a0ac to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/41241cc2395148848a6a8a8cc5e4a0ac 2024-11-13T18:14:40,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/65779f7185c7442290d515b6c926e5ec to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/65779f7185c7442290d515b6c926e5ec 2024-11-13T18:14:40,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/301723cab32b4ac7bc39553fdba223d4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/301723cab32b4ac7bc39553fdba223d4 2024-11-13T18:14:40,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/46993f4863c440f2abc2b85b405a0c23 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/46993f4863c440f2abc2b85b405a0c23 2024-11-13T18:14:40,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d23125d92c7b4e1c9643295d185052c6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/d23125d92c7b4e1c9643295d185052c6 2024-11-13T18:14:40,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/df67241fdeb340a5914cc5d32ec478bf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/df67241fdeb340a5914cc5d32ec478bf 2024-11-13T18:14:40,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/a883fe7e82d44f61bff2fa441134253c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/a883fe7e82d44f61bff2fa441134253c 2024-11-13T18:14:40,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c0f6b9f1581c42049870440daede4e33 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c0f6b9f1581c42049870440daede4e33 2024-11-13T18:14:40,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b4ffcbb0601e47f7ac0b080e08974eb3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b4ffcbb0601e47f7ac0b080e08974eb3 2024-11-13T18:14:40,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e2ebcc98fb324bcdbc6719c64b990c3e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e2ebcc98fb324bcdbc6719c64b990c3e 2024-11-13T18:14:40,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/16b96be3596f44f49204dd84bbb5a65d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/16b96be3596f44f49204dd84bbb5a65d 2024-11-13T18:14:40,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f9e05a81120a4396bf2cfb962f76f0cd to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/f9e05a81120a4396bf2cfb962f76f0cd 2024-11-13T18:14:40,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ac5edbad0eec454892137e4d17a1e25c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/ac5edbad0eec454892137e4d17a1e25c 2024-11-13T18:14:40,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e01eb6cf4c99430cbfecf9cfa796e194 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/e01eb6cf4c99430cbfecf9cfa796e194 2024-11-13T18:14:40,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b945d67651a54c9ebe25c246ff677277 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/b945d67651a54c9ebe25c246ff677277 2024-11-13T18:14:40,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/894b3218fb7149b7bc985b7ae5d4f6f3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/894b3218fb7149b7bc985b7ae5d4f6f3 2024-11-13T18:14:40,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7faaee0e166d4c0fbf5d4937095bad37 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/7faaee0e166d4c0fbf5d4937095bad37 2024-11-13T18:14:40,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/47e7ca6a1b4147eb804378bc27166598, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6650f4130ae3486a83a5c4c7cf64b59f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/9b40816f72b2411bb2c1f3d144b1ab66, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1db638ee9ca54b37ab3dc9467b37f7c0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/291da3d0a3ff41438e0327117c1f8206, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1203874cf80b464d807122dc7df3788c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/8765ccdc6dd140c2b78722d57da18f55, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e8f788ded6224715980e93fa5a133a10, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/806a9b3ab5c0411984dd5bbb927dd923, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/38b40421eee24f549e66e49393cbaf75, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ac81fabb632a474db18bc8add5286440, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/24b2da40bfc94e419f03d5ec9888d7fc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/02928dabe32d446a8bcc69c71291cdf1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5092c21c9514433ab4f405d478fa6434, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7caa5894c3bf42f4b85c1a8ff6870499, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/21083a16fcdd4b19904a8a5e5e68aaf6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b30d135c25834411b9ad67e724313439, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7a9fa5588dfa4afd80ace47fb74f0a3b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6a4999a3f13d4ec59c515d378c122916, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ee684167db664ce190e41b67e46f513d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6105fb61a41644e4b8d694b6645dba62, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/67e0d515d51a4bb6ba08fcbbfcd73e9a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/70fe8ef61e364714a57e8a19fab7eef1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6ee1c6a26f5044b7b772566a3a1c5579, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e13ecc72f9be4046bcf7a9d39e0b2cd9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/49097970024747818780b9688cc3f86a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/25edf1f53504415e9dff70a873177c77, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/cc6fefd6c0d94e108b2ed38e94959c2f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c6697294be4b4d1fb7c7e94ed7cb9ab0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c9362673870c4a65a1cb34b2462a6e07, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/69189ad9ae2340f793b53463c926b570, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/12fb24fd4dcc45f3822bf476fa78505c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/16c9e5492afe41219599531ce6445a77, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1757892a0faf4856ab64dedd035b41f1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/91e800e1220541b5beb05c8774a9a665, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/53c246ad29194f70b2d8bfb2a1c1ef48, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/2af106b1da3d4ee4a6b4792b32ec7984, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b75a57d2c2f04918973b709cd631b95b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d1e56c0f06444bc1bb7e223f48e43256, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d90a88fe27954a799ccb1bcb9196a785, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/23d483aca961452c8fc11748878d310f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/59c2e83834b24d3e86527ed1d72637cc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5bea2d95be9a4849874bf6e2626cfc9e] to archive 2024-11-13T18:14:40,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:14:40,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/47e7ca6a1b4147eb804378bc27166598 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/47e7ca6a1b4147eb804378bc27166598 2024-11-13T18:14:40,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6650f4130ae3486a83a5c4c7cf64b59f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6650f4130ae3486a83a5c4c7cf64b59f 2024-11-13T18:14:40,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/9b40816f72b2411bb2c1f3d144b1ab66 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/9b40816f72b2411bb2c1f3d144b1ab66 2024-11-13T18:14:40,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1db638ee9ca54b37ab3dc9467b37f7c0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1db638ee9ca54b37ab3dc9467b37f7c0 2024-11-13T18:14:40,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/291da3d0a3ff41438e0327117c1f8206 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/291da3d0a3ff41438e0327117c1f8206 2024-11-13T18:14:40,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1203874cf80b464d807122dc7df3788c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1203874cf80b464d807122dc7df3788c 2024-11-13T18:14:40,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/8765ccdc6dd140c2b78722d57da18f55 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/8765ccdc6dd140c2b78722d57da18f55 2024-11-13T18:14:40,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e8f788ded6224715980e93fa5a133a10 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e8f788ded6224715980e93fa5a133a10 2024-11-13T18:14:40,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/806a9b3ab5c0411984dd5bbb927dd923 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/806a9b3ab5c0411984dd5bbb927dd923 2024-11-13T18:14:40,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/38b40421eee24f549e66e49393cbaf75 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/38b40421eee24f549e66e49393cbaf75 2024-11-13T18:14:40,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ac81fabb632a474db18bc8add5286440 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ac81fabb632a474db18bc8add5286440 2024-11-13T18:14:40,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/24b2da40bfc94e419f03d5ec9888d7fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/24b2da40bfc94e419f03d5ec9888d7fc 2024-11-13T18:14:40,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/02928dabe32d446a8bcc69c71291cdf1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/02928dabe32d446a8bcc69c71291cdf1 2024-11-13T18:14:40,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5092c21c9514433ab4f405d478fa6434 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5092c21c9514433ab4f405d478fa6434 2024-11-13T18:14:40,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7caa5894c3bf42f4b85c1a8ff6870499 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7caa5894c3bf42f4b85c1a8ff6870499 2024-11-13T18:14:40,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/21083a16fcdd4b19904a8a5e5e68aaf6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/21083a16fcdd4b19904a8a5e5e68aaf6 2024-11-13T18:14:40,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b30d135c25834411b9ad67e724313439 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b30d135c25834411b9ad67e724313439 2024-11-13T18:14:40,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7a9fa5588dfa4afd80ace47fb74f0a3b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/7a9fa5588dfa4afd80ace47fb74f0a3b 2024-11-13T18:14:40,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6a4999a3f13d4ec59c515d378c122916 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6a4999a3f13d4ec59c515d378c122916 2024-11-13T18:14:40,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ee684167db664ce190e41b67e46f513d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/ee684167db664ce190e41b67e46f513d 2024-11-13T18:14:40,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6105fb61a41644e4b8d694b6645dba62 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6105fb61a41644e4b8d694b6645dba62 2024-11-13T18:14:40,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/67e0d515d51a4bb6ba08fcbbfcd73e9a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/67e0d515d51a4bb6ba08fcbbfcd73e9a 2024-11-13T18:14:40,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/70fe8ef61e364714a57e8a19fab7eef1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/70fe8ef61e364714a57e8a19fab7eef1 2024-11-13T18:14:40,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6ee1c6a26f5044b7b772566a3a1c5579 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/6ee1c6a26f5044b7b772566a3a1c5579 2024-11-13T18:14:40,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e13ecc72f9be4046bcf7a9d39e0b2cd9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/e13ecc72f9be4046bcf7a9d39e0b2cd9 2024-11-13T18:14:40,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/49097970024747818780b9688cc3f86a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/49097970024747818780b9688cc3f86a 2024-11-13T18:14:40,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/25edf1f53504415e9dff70a873177c77 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/25edf1f53504415e9dff70a873177c77 2024-11-13T18:14:40,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/cc6fefd6c0d94e108b2ed38e94959c2f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/cc6fefd6c0d94e108b2ed38e94959c2f 2024-11-13T18:14:40,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c6697294be4b4d1fb7c7e94ed7cb9ab0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c6697294be4b4d1fb7c7e94ed7cb9ab0 2024-11-13T18:14:40,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c9362673870c4a65a1cb34b2462a6e07 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/c9362673870c4a65a1cb34b2462a6e07 2024-11-13T18:14:40,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/69189ad9ae2340f793b53463c926b570 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/69189ad9ae2340f793b53463c926b570 2024-11-13T18:14:40,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/12fb24fd4dcc45f3822bf476fa78505c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/12fb24fd4dcc45f3822bf476fa78505c 2024-11-13T18:14:40,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/16c9e5492afe41219599531ce6445a77 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/16c9e5492afe41219599531ce6445a77 2024-11-13T18:14:40,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1757892a0faf4856ab64dedd035b41f1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/1757892a0faf4856ab64dedd035b41f1 2024-11-13T18:14:40,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/91e800e1220541b5beb05c8774a9a665 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/91e800e1220541b5beb05c8774a9a665 2024-11-13T18:14:40,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/53c246ad29194f70b2d8bfb2a1c1ef48 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/53c246ad29194f70b2d8bfb2a1c1ef48 2024-11-13T18:14:40,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/2af106b1da3d4ee4a6b4792b32ec7984 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/2af106b1da3d4ee4a6b4792b32ec7984 2024-11-13T18:14:40,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b75a57d2c2f04918973b709cd631b95b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/b75a57d2c2f04918973b709cd631b95b 2024-11-13T18:14:40,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d1e56c0f06444bc1bb7e223f48e43256 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d1e56c0f06444bc1bb7e223f48e43256 2024-11-13T18:14:40,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d90a88fe27954a799ccb1bcb9196a785 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/d90a88fe27954a799ccb1bcb9196a785 2024-11-13T18:14:40,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/23d483aca961452c8fc11748878d310f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/23d483aca961452c8fc11748878d310f 2024-11-13T18:14:40,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/59c2e83834b24d3e86527ed1d72637cc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/59c2e83834b24d3e86527ed1d72637cc 2024-11-13T18:14:40,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5bea2d95be9a4849874bf6e2626cfc9e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/5bea2d95be9a4849874bf6e2626cfc9e 2024-11-13T18:14:40,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/2784fe404ab0402d9fcbdcfedfde6956, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3b4f0dcab9a341909d743e31d4271727, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/38af54a893c448f5bf7fc575a5027155, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c6b09e5e2e034f4fa5bf72d18768749e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c3ee815de299451c9770c5f3a65b7f16, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41440fd9fafc488e8b6e7a92b3012429, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/9bda1451005a4609921d0122a0484d4f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/e497b9f8714d4cb588aea88016f6860a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b37a7b37b6844770ad0601021ae0afb1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ed40bca2e08d426784fdab58cfbb5e33, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/62f142918fb8417db083d2758216ad13, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/13204f8d84e34fd3b366b6b604a4e6cf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/553edf81b563480fbdf25a9a455e6c75, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/dfb02d62bd1c46889638d4f697859e53, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41e99a5c7c6e4ad2a15f2258e6378f72, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f656f02eb3a1483baf834089d9ef4df2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f3516d95ed3d4c558e7a9e33a0cea246, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b79acdc0bc354dee9575867b1a0c2ecf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/12389b814f7e45f0b5503e5683caceeb, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/6017f5051a5b41f58d8a89fb519be1d3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ad10b6abc5d446cdaed30b1eafdb4053, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/caf95196d3e2497a903f8f5fbc3ec21a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/739c39b7a73845da8a69fb3bd319343d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/123e2ae3194549b48fd789b6bfba6b74, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3288390634194c54bad05e057771bc81, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/7b69e67612f14f53aaf7d1701cb93ac4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a6ea65c536d14e378d4c25b37853c9c8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b9569645a2fb4b63bcfceca5e4799645, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/31ca373cbd5c43c391aed6a59a447f26, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/67849dcfb43f497c8ad86de342fd436c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a287c903bebd4c79b425170a7b641049, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/d368947c0f694f06815a5392fba8700d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f493944af174436cb87498a1f5ef18d9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f1884dff55854e8eb4980218f371d6ac, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/aa8c00ff68944b3f887179c492368e0c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f45a9f260af141e99f95da312ba01022, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/94dca408274b4c89b2b8a3501ca7e9b4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/689cf413d7af4c2b98f494932d597ee3, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a851d59387e543eb813d737b35e61321, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0729d2fb3b4e45fd965eb31c832b138d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0cdd3c89c654407aa698cf63eb9c358e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/cc9a625802dc4c62a829b85fca45656e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b8042a4f85d24d768f7fb43922c95ac9] to archive 2024-11-13T18:14:40,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:14:40,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/2784fe404ab0402d9fcbdcfedfde6956 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/2784fe404ab0402d9fcbdcfedfde6956 2024-11-13T18:14:40,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3b4f0dcab9a341909d743e31d4271727 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3b4f0dcab9a341909d743e31d4271727 2024-11-13T18:14:40,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/38af54a893c448f5bf7fc575a5027155 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/38af54a893c448f5bf7fc575a5027155 2024-11-13T18:14:40,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c6b09e5e2e034f4fa5bf72d18768749e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c6b09e5e2e034f4fa5bf72d18768749e 2024-11-13T18:14:40,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c3ee815de299451c9770c5f3a65b7f16 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/c3ee815de299451c9770c5f3a65b7f16 2024-11-13T18:14:40,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41440fd9fafc488e8b6e7a92b3012429 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41440fd9fafc488e8b6e7a92b3012429 2024-11-13T18:14:40,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/9bda1451005a4609921d0122a0484d4f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/9bda1451005a4609921d0122a0484d4f 2024-11-13T18:14:40,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/e497b9f8714d4cb588aea88016f6860a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/e497b9f8714d4cb588aea88016f6860a 2024-11-13T18:14:40,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b37a7b37b6844770ad0601021ae0afb1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b37a7b37b6844770ad0601021ae0afb1 2024-11-13T18:14:40,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ed40bca2e08d426784fdab58cfbb5e33 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ed40bca2e08d426784fdab58cfbb5e33 2024-11-13T18:14:40,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/62f142918fb8417db083d2758216ad13 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/62f142918fb8417db083d2758216ad13 2024-11-13T18:14:40,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/13204f8d84e34fd3b366b6b604a4e6cf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/13204f8d84e34fd3b366b6b604a4e6cf 2024-11-13T18:14:40,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/553edf81b563480fbdf25a9a455e6c75 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/553edf81b563480fbdf25a9a455e6c75 2024-11-13T18:14:40,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/dfb02d62bd1c46889638d4f697859e53 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/dfb02d62bd1c46889638d4f697859e53 2024-11-13T18:14:40,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41e99a5c7c6e4ad2a15f2258e6378f72 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/41e99a5c7c6e4ad2a15f2258e6378f72 2024-11-13T18:14:40,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f656f02eb3a1483baf834089d9ef4df2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f656f02eb3a1483baf834089d9ef4df2 2024-11-13T18:14:40,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f3516d95ed3d4c558e7a9e33a0cea246 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f3516d95ed3d4c558e7a9e33a0cea246 2024-11-13T18:14:40,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b79acdc0bc354dee9575867b1a0c2ecf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b79acdc0bc354dee9575867b1a0c2ecf 2024-11-13T18:14:40,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/12389b814f7e45f0b5503e5683caceeb to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/12389b814f7e45f0b5503e5683caceeb 2024-11-13T18:14:40,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/6017f5051a5b41f58d8a89fb519be1d3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/6017f5051a5b41f58d8a89fb519be1d3 2024-11-13T18:14:40,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ad10b6abc5d446cdaed30b1eafdb4053 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/ad10b6abc5d446cdaed30b1eafdb4053 2024-11-13T18:14:40,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/caf95196d3e2497a903f8f5fbc3ec21a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/caf95196d3e2497a903f8f5fbc3ec21a 2024-11-13T18:14:40,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/739c39b7a73845da8a69fb3bd319343d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/739c39b7a73845da8a69fb3bd319343d 2024-11-13T18:14:40,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/123e2ae3194549b48fd789b6bfba6b74 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/123e2ae3194549b48fd789b6bfba6b74 2024-11-13T18:14:40,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3288390634194c54bad05e057771bc81 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/3288390634194c54bad05e057771bc81 2024-11-13T18:14:40,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/7b69e67612f14f53aaf7d1701cb93ac4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/7b69e67612f14f53aaf7d1701cb93ac4 2024-11-13T18:14:40,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a6ea65c536d14e378d4c25b37853c9c8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a6ea65c536d14e378d4c25b37853c9c8 2024-11-13T18:14:40,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b9569645a2fb4b63bcfceca5e4799645 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b9569645a2fb4b63bcfceca5e4799645 2024-11-13T18:14:40,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/31ca373cbd5c43c391aed6a59a447f26 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/31ca373cbd5c43c391aed6a59a447f26 2024-11-13T18:14:40,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/67849dcfb43f497c8ad86de342fd436c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/67849dcfb43f497c8ad86de342fd436c 2024-11-13T18:14:40,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a287c903bebd4c79b425170a7b641049 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a287c903bebd4c79b425170a7b641049 2024-11-13T18:14:40,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/d368947c0f694f06815a5392fba8700d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/d368947c0f694f06815a5392fba8700d 2024-11-13T18:14:40,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f493944af174436cb87498a1f5ef18d9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f493944af174436cb87498a1f5ef18d9 2024-11-13T18:14:40,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f1884dff55854e8eb4980218f371d6ac to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f1884dff55854e8eb4980218f371d6ac 2024-11-13T18:14:40,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/aa8c00ff68944b3f887179c492368e0c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/aa8c00ff68944b3f887179c492368e0c 2024-11-13T18:14:40,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f45a9f260af141e99f95da312ba01022 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/f45a9f260af141e99f95da312ba01022 2024-11-13T18:14:40,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/94dca408274b4c89b2b8a3501ca7e9b4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/94dca408274b4c89b2b8a3501ca7e9b4 2024-11-13T18:14:40,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/689cf413d7af4c2b98f494932d597ee3 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/689cf413d7af4c2b98f494932d597ee3 2024-11-13T18:14:40,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a851d59387e543eb813d737b35e61321 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/a851d59387e543eb813d737b35e61321 2024-11-13T18:14:40,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0729d2fb3b4e45fd965eb31c832b138d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0729d2fb3b4e45fd965eb31c832b138d 2024-11-13T18:14:40,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0cdd3c89c654407aa698cf63eb9c358e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/0cdd3c89c654407aa698cf63eb9c358e 2024-11-13T18:14:40,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/cc9a625802dc4c62a829b85fca45656e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/cc9a625802dc4c62a829b85fca45656e 2024-11-13T18:14:40,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b8042a4f85d24d768f7fb43922c95ac9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/b8042a4f85d24d768f7fb43922c95ac9 2024-11-13T18:14:40,613 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/recovered.edits/681.seqid, newMaxSeqId=681, maxSeqId=1 2024-11-13T18:14:40,613 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. 2024-11-13T18:14:40,614 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 278d3a3ebd6be30bccd59b5fe89f892a: Waiting for close lock at 1731521679178Running coprocessor pre-close hooks at 1731521679178Disabling compacts and flushes for region at 1731521679178Disabling writes for close at 1731521679179 (+1 ms)Obtaining lock to block concurrent updates at 1731521679179Preparing flush snapshotting stores in 278d3a3ebd6be30bccd59b5fe89f892a at 1731521679179Finished memstore snapshotting TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731521679179Flushing stores of TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a. at 1731521679180 (+1 ms)Flushing 278d3a3ebd6be30bccd59b5fe89f892a/A: creating writer at 1731521679180Flushing 278d3a3ebd6be30bccd59b5fe89f892a/A: appending metadata at 1731521679182 (+2 ms)Flushing 278d3a3ebd6be30bccd59b5fe89f892a/A: closing flushed file at 1731521679182Flushing 278d3a3ebd6be30bccd59b5fe89f892a/B: creating writer at 1731521679591 (+409 ms)Flushing 278d3a3ebd6be30bccd59b5fe89f892a/B: appending metadata at 1731521679594 (+3 ms)Flushing 278d3a3ebd6be30bccd59b5fe89f892a/B: closing flushed file at 1731521679594Flushing 278d3a3ebd6be30bccd59b5fe89f892a/C: creating writer at 1731521680003 (+409 ms)Flushing 278d3a3ebd6be30bccd59b5fe89f892a/C: appending metadata at 1731521680005 (+2 ms)Flushing 278d3a3ebd6be30bccd59b5fe89f892a/C: closing flushed file at 1731521680005Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74efdf40: reopening flushed file at 1731521680414 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@545d5aa3: reopening flushed file at 1731521680417 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@541f2287: reopening flushed file at 1731521680420 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 278d3a3ebd6be30bccd59b5fe89f892a in 1245ms, sequenceid=678, compaction requested=false at 1731521680424 (+4 ms)Writing region close event to WAL at 1731521680609 (+185 ms)Running coprocessor post-close hooks at 1731521680613 (+4 ms)Closed at 1731521680613 2024-11-13T18:14:40,620 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=278d3a3ebd6be30bccd59b5fe89f892a, regionState=CLOSED 2024-11-13T18:14:40,621 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:40,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:14:40,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-13T18:14:40,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 278d3a3ebd6be30bccd59b5fe89f892a, server=fda96cf025dc,44483,1731521526892 in 1.6000 sec 2024-11-13T18:14:40,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-13T18:14:40,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=278d3a3ebd6be30bccd59b5fe89f892a, UNASSIGN in 1.6040 sec 2024-11-13T18:14:40,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-13T18:14:40,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6090 sec 2024-11-13T18:14:40,634 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521680634"}]},"ts":"1731521680634"} 2024-11-13T18:14:40,636 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:14:40,636 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:14:40,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6200 sec 2024-11-13T18:14:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T18:14:41,164 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:41,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:14:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:41,167 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T18:14:41,167 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:41,169 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:41,171 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/recovered.edits] 2024-11-13T18:14:41,173 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/4c07ed6eb1084ca9a8bfa0b03270a8d5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/4c07ed6eb1084ca9a8bfa0b03270a8d5 2024-11-13T18:14:41,174 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c8f6e1123db54f19be3f3b562995d20d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/A/c8f6e1123db54f19be3f3b562995d20d 2024-11-13T18:14:41,176 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/cd370ddf10a84197888bc8cca2363760 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/cd370ddf10a84197888bc8cca2363760 2024-11-13T18:14:41,177 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/dcf5f464d1ea42dc83d6d07c52718904 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/B/dcf5f464d1ea42dc83d6d07c52718904 2024-11-13T18:14:41,185 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/454c4a5442ec4e25a076589060c212d8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/454c4a5442ec4e25a076589060c212d8 2024-11-13T18:14:41,186 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/92f1b3f2b4f044aab4ccc5fc645ea34e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/C/92f1b3f2b4f044aab4ccc5fc645ea34e 2024-11-13T18:14:41,188 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/recovered.edits/681.seqid to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a/recovered.edits/681.seqid 2024-11-13T18:14:41,188 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/278d3a3ebd6be30bccd59b5fe89f892a 2024-11-13T18:14:41,188 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:14:41,190 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:41,192 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:14:41,194 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:14:41,195 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:41,195 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:14:41,195 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731521681195"}]},"ts":"9223372036854775807"} 2024-11-13T18:14:41,197 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:14:41,197 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 278d3a3ebd6be30bccd59b5fe89f892a, NAME => 'TestAcidGuarantees,,1731521651857.278d3a3ebd6be30bccd59b5fe89f892a.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:14:41,197 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:14:41,197 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731521681197"}]},"ts":"9223372036854775807"} 2024-11-13T18:14:41,199 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:14:41,199 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:41,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-13T18:14:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T18:14:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:14:41,274 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:41,283 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 252), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=879 (was 778) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=279 (was 899) 2024-11-13T18:14:41,293 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=879, ProcessCount=11, AvailableMemoryMB=278 2024-11-13T18:14:41,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:14:41,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:14:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:41,297 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:14:41,297 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:41,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-13T18:14:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:14:41,298 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:14:41,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742487_1663 (size=1061) 2024-11-13T18:14:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:14:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:14:41,708 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1 2024-11-13T18:14:41,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742488_1664 (size=53) 2024-11-13T18:14:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:14:42,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:14:42,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a5a3ed238deaa2c83ee0dd5f472061c5, disabling compactions & flushes 2024-11-13T18:14:42,115 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:42,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:42,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. after waiting 0 ms 2024-11-13T18:14:42,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:42,115 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:42,116 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a5a3ed238deaa2c83ee0dd5f472061c5: Waiting for close lock at 1731521682115Disabling compacts and flushes for region at 1731521682115Disabling writes for close at 1731521682115Writing region close event to WAL at 1731521682115Closed at 1731521682115 2024-11-13T18:14:42,117 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:14:42,117 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731521682117"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521682117"}]},"ts":"1731521682117"} 2024-11-13T18:14:42,118 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:14:42,119 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:14:42,119 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521682119"}]},"ts":"1731521682119"} 2024-11-13T18:14:42,120 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:14:42,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, ASSIGN}] 2024-11-13T18:14:42,121 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, ASSIGN 2024-11-13T18:14:42,121 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, ASSIGN; state=OFFLINE, location=fda96cf025dc,44483,1731521526892; forceNewPlan=false, retain=false 2024-11-13T18:14:42,272 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=a5a3ed238deaa2c83ee0dd5f472061c5, regionState=OPENING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:42,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, ASSIGN because future has completed 2024-11-13T18:14:42,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:14:42,429 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:42,429 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:14:42,429 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,429 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:14:42,430 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,430 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,431 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,432 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:14:42,432 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5a3ed238deaa2c83ee0dd5f472061c5 columnFamilyName A 2024-11-13T18:14:42,432 DEBUG [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:42,432 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(327): Store=a5a3ed238deaa2c83ee0dd5f472061c5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:14:42,432 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,433 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:14:42,433 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5a3ed238deaa2c83ee0dd5f472061c5 columnFamilyName B 2024-11-13T18:14:42,433 DEBUG [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:14:42,434 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(327): Store=a5a3ed238deaa2c83ee0dd5f472061c5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:14:42,434 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,435 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:14:42,435 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5a3ed238deaa2c83ee0dd5f472061c5 columnFamilyName C 2024-11-13T18:14:42,435 DEBUG [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:42,435 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(327): Store=a5a3ed238deaa2c83ee0dd5f472061c5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:14:42,435 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:42,435 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,436 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,436 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,437 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,437 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,437 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:14:42,438 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,439 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:14:42,440 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened a5a3ed238deaa2c83ee0dd5f472061c5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63895213, jitterRate=-0.047887131571769714}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:14:42,440 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:42,440 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for a5a3ed238deaa2c83ee0dd5f472061c5: Running coprocessor pre-open hook at 1731521682430Writing region info on filesystem at 1731521682430Initializing all the Stores at 1731521682430Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521682430Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521682431 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521682431Cleaning up temporary data from old regions at 1731521682437 (+6 ms)Running coprocessor post-open hooks at 1731521682440 (+3 ms)Region opened successfully at 1731521682440 2024-11-13T18:14:42,441 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., pid=142, masterSystemTime=1731521682426 2024-11-13T18:14:42,442 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:42,443 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:42,443 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=a5a3ed238deaa2c83ee0dd5f472061c5, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:42,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:14:42,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-13T18:14:42,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 in 171 msec 2024-11-13T18:14:42,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-13T18:14:42,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, ASSIGN in 326 msec 2024-11-13T18:14:42,448 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:14:42,448 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521682448"}]},"ts":"1731521682448"} 2024-11-13T18:14:42,449 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:14:42,450 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:14:42,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-13T18:14:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:14:43,445 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:43,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d88bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:43,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:43,447 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:43,448 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@530227f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:43,451 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4076d1cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:43,452 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:43,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:43,455 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:14:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,37165,1731521525941 2024-11-13T18:14:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d583fb1 2024-11-13T18:14:43,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:14:43,456 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:14:43,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:14:43,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:14:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T18:14:43,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742489_1665 (size=1097) 2024-11-13T18:14:43,881 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T18:14:43,881 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T18:14:43,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:14:43,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, REOPEN/MOVE}] 2024-11-13T18:14:43,887 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, REOPEN/MOVE 2024-11-13T18:14:43,887 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=a5a3ed238deaa2c83ee0dd5f472061c5, regionState=CLOSING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:43,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, REOPEN/MOVE because future has completed 2024-11-13T18:14:43,889 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:14:43,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:14:44,041 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,041 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:14:44,041 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing a5a3ed238deaa2c83ee0dd5f472061c5, disabling compactions & flushes 2024-11-13T18:14:44,041 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,041 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,041 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. after waiting 0 ms 2024-11-13T18:14:44,042 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,054 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:14:44,055 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,055 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for a5a3ed238deaa2c83ee0dd5f472061c5: Waiting for close lock at 1731521684041Running coprocessor pre-close hooks at 1731521684041Disabling compacts and flushes for region at 1731521684041Disabling writes for close at 1731521684041Writing region close event to WAL at 1731521684049 (+8 ms)Running coprocessor post-close hooks at 1731521684054 (+5 ms)Closed at 1731521684055 (+1 ms) 2024-11-13T18:14:44,055 WARN [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: a5a3ed238deaa2c83ee0dd5f472061c5 to self. 2024-11-13T18:14:44,057 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,057 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=a5a3ed238deaa2c83ee0dd5f472061c5, regionState=CLOSED 2024-11-13T18:14:44,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:14:44,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-13T18:14:44,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 in 170 msec 2024-11-13T18:14:44,061 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, REOPEN/MOVE; state=CLOSED, location=fda96cf025dc,44483,1731521526892; forceNewPlan=false, retain=true 2024-11-13T18:14:44,212 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=a5a3ed238deaa2c83ee0dd5f472061c5, regionState=OPENING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, REOPEN/MOVE because future has completed 2024-11-13T18:14:44,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:14:44,375 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,375 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:14:44,375 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,375 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:14:44,375 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,375 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,376 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,377 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:14:44,377 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5a3ed238deaa2c83ee0dd5f472061c5 columnFamilyName A 2024-11-13T18:14:44,378 DEBUG [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:44,378 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(327): Store=a5a3ed238deaa2c83ee0dd5f472061c5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:14:44,378 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,379 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:14:44,379 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5a3ed238deaa2c83ee0dd5f472061c5 columnFamilyName B 2024-11-13T18:14:44,379 DEBUG [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:44,379 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(327): Store=a5a3ed238deaa2c83ee0dd5f472061c5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:14:44,379 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,380 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:14:44,380 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5a3ed238deaa2c83ee0dd5f472061c5 columnFamilyName C 2024-11-13T18:14:44,380 DEBUG [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:44,380 INFO [StoreOpener-a5a3ed238deaa2c83ee0dd5f472061c5-1 {}] regionserver.HStore(327): Store=a5a3ed238deaa2c83ee0dd5f472061c5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:14:44,380 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,381 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,381 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,382 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,383 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,383 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,384 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:14:44,385 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,385 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened a5a3ed238deaa2c83ee0dd5f472061c5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72393333, jitterRate=0.07874472439289093}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:14:44,385 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,386 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for a5a3ed238deaa2c83ee0dd5f472061c5: Running coprocessor pre-open hook at 1731521684375Writing region info on filesystem at 1731521684375Initializing all the Stores at 1731521684376 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521684376Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521684376Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521684376Cleaning up temporary data from old regions at 1731521684383 (+7 ms)Running coprocessor post-open hooks at 1731521684385 (+2 ms)Region opened successfully at 1731521684386 (+1 ms) 2024-11-13T18:14:44,386 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., pid=147, masterSystemTime=1731521684372 2024-11-13T18:14:44,388 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,388 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,388 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=a5a3ed238deaa2c83ee0dd5f472061c5, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:14:44,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-13T18:14:44,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 in 170 msec 2024-11-13T18:14:44,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-13T18:14:44,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, REOPEN/MOVE in 506 msec 2024-11-13T18:14:44,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-13T18:14:44,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-11-13T18:14:44,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 937 msec 2024-11-13T18:14:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-13T18:14:44,398 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:44,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4ca04b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:44,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:44,400 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@481bd34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:44,401 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:44,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f55504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a2247f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:44,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:44,403 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd818b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:44,404 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@543a3346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@524cf355, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:44,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:44,406 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1895ebd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:44,407 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d5f958, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fde4b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:44,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:44,408 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ace8a09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:44,409 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217c06b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1e1010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:44,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:44,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de1d51a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:44,412 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277ff29e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2428e86b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:44,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:44,414 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58bbf76d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:44,415 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f40a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a01bf38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:44,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:44,416 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5087874a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:44,417 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa1e14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45dc52f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:44,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:44,419 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3afea6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:44,420 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@438d38ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e13bf73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:44,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:44,421 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53c652b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:44,423 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a13e32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769ef175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,37165,-1 for getting cluster id 2024-11-13T18:14:44,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:14:44,424 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab' 2024-11-13T18:14:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:14:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c89f0ca1-8d81-4a58-bfc6-a5f32bc2f5ab" 2024-11-13T18:14:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b193c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,37165,-1] 2024-11-13T18:14:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:14:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:14:44,426 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:14:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be197db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:14:44,427 DEBUG [Thread-2914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:44,427 DEBUG [Thread-2916 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:44,427 DEBUG [Thread-2925 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:44,427 DEBUG [Thread-2908 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:44,427 DEBUG [Thread-2927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:44,428 DEBUG [Thread-2912 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:44,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:44,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:44,429 DEBUG [Thread-2910 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:44,429 DEBUG [Thread-2919 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:44,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:44,430 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:44,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:44,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:44,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:44,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:44,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:14:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:14:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:14:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:14:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:14:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:14:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:14:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-13T18:14:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:14:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:14:44,437 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:44,438 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:44,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:44,441 DEBUG [Thread-2923 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:44,442 DEBUG [Thread-2921 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:14:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:14:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44483,1731521526892, seqNum=-1] 2024-11-13T18:14:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:14:44,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:44,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:44,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:44,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:44,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:14:44,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:44,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:44,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:14:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5] 2024-11-13T18:14:44,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521744464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521744466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521744466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113402c8ca642674dc3b7f2ee3d0e941dcf_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521684441/Put/seqid=0 2024-11-13T18:14:44,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521744468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521744468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742490_1666 (size=12154) 2024-11-13T18:14:44,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:44,486 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113402c8ca642674dc3b7f2ee3d0e941dcf_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113402c8ca642674dc3b7f2ee3d0e941dcf_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:44,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/9d0b304ca9ee47e18556111db443884e, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:44,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/9d0b304ca9ee47e18556111db443884e is 175, key is test_row_0/A:col10/1731521684441/Put/seqid=0 2024-11-13T18:14:44,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742491_1667 (size=30955) 2024-11-13T18:14:44,496 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/9d0b304ca9ee47e18556111db443884e 2024-11-13T18:14:44,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/1372339871af4324b6063a430896fda5 is 50, key is test_row_0/B:col10/1731521684441/Put/seqid=0 2024-11-13T18:14:44,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742492_1668 (size=12001) 2024-11-13T18:14:44,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/1372339871af4324b6063a430896fda5 2024-11-13T18:14:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:14:44,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/bf5f1562d53c47e9a937950e82f7d53b is 50, key is test_row_0/C:col10/1731521684441/Put/seqid=0 2024-11-13T18:14:44,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742493_1669 (size=12001) 2024-11-13T18:14:44,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521744574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521744574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521744574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521744584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521744584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:14:44,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:44,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:44,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:44,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:14:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:14:44,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521744784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521744784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521744785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521744794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:44,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521744794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:44,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:14:44,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:44,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:44,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:44,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:44,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:44,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/bf5f1562d53c47e9a937950e82f7d53b 2024-11-13T18:14:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/9d0b304ca9ee47e18556111db443884e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/9d0b304ca9ee47e18556111db443884e 2024-11-13T18:14:45,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/9d0b304ca9ee47e18556111db443884e, entries=150, sequenceid=16, filesize=30.2 K 2024-11-13T18:14:45,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/1372339871af4324b6063a430896fda5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1372339871af4324b6063a430896fda5 2024-11-13T18:14:45,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1372339871af4324b6063a430896fda5, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T18:14:45,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/bf5f1562d53c47e9a937950e82f7d53b as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/bf5f1562d53c47e9a937950e82f7d53b 2024-11-13T18:14:45,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/bf5f1562d53c47e9a937950e82f7d53b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T18:14:45,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a5a3ed238deaa2c83ee0dd5f472061c5 in 575ms, sequenceid=16, compaction requested=false 2024-11-13T18:14:45,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:45,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:14:45,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:45,058 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:14:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:45,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111329dceebe04ef4afb977ca5ce3941d551_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521684450/Put/seqid=0 2024-11-13T18:14:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:14:45,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742494_1670 (size=12154) 2024-11-13T18:14:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:45,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:45,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521745104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521745105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521745106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521745106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521745107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521745214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521745214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521745217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521745424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521745424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521745424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:45,487 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111329dceebe04ef4afb977ca5ce3941d551_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111329dceebe04ef4afb977ca5ce3941d551_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:45,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/74ee2570a89c445e8de1c1b348d99850, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:45,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/74ee2570a89c445e8de1c1b348d99850 is 175, key is test_row_0/A:col10/1731521684450/Put/seqid=0 2024-11-13T18:14:45,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742495_1671 (size=30955) 2024-11-13T18:14:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:14:45,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521745622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521745622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521745737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521745737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521745737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:45,894 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/74ee2570a89c445e8de1c1b348d99850 2024-11-13T18:14:45,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/4ec5d97f53ec4bf5aae0b4cc4e5a011e is 50, key is test_row_0/B:col10/1731521684450/Put/seqid=0 2024-11-13T18:14:45,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742496_1672 (size=12001) 2024-11-13T18:14:45,934 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/4ec5d97f53ec4bf5aae0b4cc4e5a011e 2024-11-13T18:14:45,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/9a1e6b7310394471a4e196b40d8f3a68 is 50, key is test_row_0/C:col10/1731521684450/Put/seqid=0 2024-11-13T18:14:45,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742497_1673 (size=12001) 2024-11-13T18:14:46,141 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:14:46,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521746244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:46,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521746244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:46,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521746254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:46,353 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/9a1e6b7310394471a4e196b40d8f3a68 2024-11-13T18:14:46,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/74ee2570a89c445e8de1c1b348d99850 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/74ee2570a89c445e8de1c1b348d99850 2024-11-13T18:14:46,362 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/74ee2570a89c445e8de1c1b348d99850, entries=150, sequenceid=36, filesize=30.2 K 2024-11-13T18:14:46,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/4ec5d97f53ec4bf5aae0b4cc4e5a011e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/4ec5d97f53ec4bf5aae0b4cc4e5a011e 2024-11-13T18:14:46,366 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/4ec5d97f53ec4bf5aae0b4cc4e5a011e, entries=150, sequenceid=36, filesize=11.7 K 2024-11-13T18:14:46,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/9a1e6b7310394471a4e196b40d8f3a68 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9a1e6b7310394471a4e196b40d8f3a68 2024-11-13T18:14:46,372 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9a1e6b7310394471a4e196b40d8f3a68, entries=150, sequenceid=36, filesize=11.7 K 2024-11-13T18:14:46,373 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a5a3ed238deaa2c83ee0dd5f472061c5 in 1315ms, sequenceid=36, compaction requested=false 2024-11-13T18:14:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-13T18:14:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-13T18:14:46,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-13T18:14:46,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9370 sec 2024-11-13T18:14:46,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.9490 sec 2024-11-13T18:14:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:14:46,595 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:46,595 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-13T18:14:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:14:46,597 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:46,598 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:46,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:46,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:14:46,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:46,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:46,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:46,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:46,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:46,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:46,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131a07662acfd24d4298878fa023e4fd95_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521685105/Put/seqid=0 2024-11-13T18:14:46,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742498_1674 (size=14594) 2024-11-13T18:14:46,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:46,659 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131a07662acfd24d4298878fa023e4fd95_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131a07662acfd24d4298878fa023e4fd95_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:46,660 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/0c3522f4a87948bdade2b7c1bd8b3315, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:46,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/0c3522f4a87948bdade2b7c1bd8b3315 is 175, key is test_row_0/A:col10/1731521685105/Put/seqid=0 2024-11-13T18:14:46,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742499_1675 (size=39549) 2024-11-13T18:14:46,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521746690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:46,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521746690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:14:46,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:14:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:46,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:46,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521746804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:46,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521746804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:46,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:14:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:46,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:14:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521747014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521747014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:14:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:47,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:47,073 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/0c3522f4a87948bdade2b7c1bd8b3315 2024-11-13T18:14:47,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/9c9e63ccdbcd474c98022e2364b995e7 is 50, key is test_row_0/B:col10/1731521685105/Put/seqid=0 2024-11-13T18:14:47,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742500_1676 (size=12001) 2024-11-13T18:14:47,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/9c9e63ccdbcd474c98022e2364b995e7 2024-11-13T18:14:47,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/e21a1d784d0b4df09e2ecabbf4a23191 is 50, key is test_row_0/C:col10/1731521685105/Put/seqid=0 2024-11-13T18:14:47,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742501_1677 (size=12001) 2024-11-13T18:14:47,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/e21a1d784d0b4df09e2ecabbf4a23191 2024-11-13T18:14:47,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/0c3522f4a87948bdade2b7c1bd8b3315 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/0c3522f4a87948bdade2b7c1bd8b3315 2024-11-13T18:14:47,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/0c3522f4a87948bdade2b7c1bd8b3315, entries=200, sequenceid=49, filesize=38.6 K 2024-11-13T18:14:47,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/9c9e63ccdbcd474c98022e2364b995e7 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c9e63ccdbcd474c98022e2364b995e7 2024-11-13T18:14:47,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c9e63ccdbcd474c98022e2364b995e7, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T18:14:47,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/e21a1d784d0b4df09e2ecabbf4a23191 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e21a1d784d0b4df09e2ecabbf4a23191 2024-11-13T18:14:47,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e21a1d784d0b4df09e2ecabbf4a23191, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T18:14:47,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a5a3ed238deaa2c83ee0dd5f472061c5 in 549ms, sequenceid=49, compaction requested=true 2024-11-13T18:14:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:47,185 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:47,187 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:47,187 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/A is initiating minor compaction (all files) 2024-11-13T18:14:47,187 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/A in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:47,188 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/9d0b304ca9ee47e18556111db443884e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/74ee2570a89c445e8de1c1b348d99850, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/0c3522f4a87948bdade2b7c1bd8b3315] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=99.1 K 2024-11-13T18:14:47,188 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:47,188 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/9d0b304ca9ee47e18556111db443884e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/74ee2570a89c445e8de1c1b348d99850, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/0c3522f4a87948bdade2b7c1bd8b3315] 2024-11-13T18:14:47,188 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d0b304ca9ee47e18556111db443884e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731521684441 2024-11-13T18:14:47,188 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:47,188 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ee2570a89c445e8de1c1b348d99850, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731521684450 2024-11-13T18:14:47,189 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c3522f4a87948bdade2b7c1bd8b3315, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731521685096 2024-11-13T18:14:47,190 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:47,190 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/B is initiating minor compaction (all files) 2024-11-13T18:14:47,190 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/B in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:47,190 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1372339871af4324b6063a430896fda5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/4ec5d97f53ec4bf5aae0b4cc4e5a011e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c9e63ccdbcd474c98022e2364b995e7] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=35.2 K 2024-11-13T18:14:47,191 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1372339871af4324b6063a430896fda5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731521684441 2024-11-13T18:14:47,191 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ec5d97f53ec4bf5aae0b4cc4e5a011e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731521684450 2024-11-13T18:14:47,192 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c9e63ccdbcd474c98022e2364b995e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731521685101 2024-11-13T18:14:47,200 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:47,204 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#B#compaction#582 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:47,205 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/305935362a5841c1bdbea62270831183 is 50, key is test_row_0/B:col10/1731521685105/Put/seqid=0 2024-11-13T18:14:47,207 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138af49e4fc3114e70a467d9dfbd8905a5_a5a3ed238deaa2c83ee0dd5f472061c5 store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:47,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:14:47,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:47,210 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:14:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:47,210 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138af49e4fc3114e70a467d9dfbd8905a5_a5a3ed238deaa2c83ee0dd5f472061c5, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:47,211 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138af49e4fc3114e70a467d9dfbd8905a5_a5a3ed238deaa2c83ee0dd5f472061c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:14:47,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742502_1678 (size=12104) 2024-11-13T18:14:47,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111395246000f0f643898803e8d2a856f1c5_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521686687/Put/seqid=0 2024-11-13T18:14:47,240 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/305935362a5841c1bdbea62270831183 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/305935362a5841c1bdbea62270831183 2024-11-13T18:14:47,252 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/B of a5a3ed238deaa2c83ee0dd5f472061c5 into 305935362a5841c1bdbea62270831183(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:47,252 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:47,252 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/B, priority=13, startTime=1731521687185; duration=0sec 2024-11-13T18:14:47,252 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:47,252 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:14:47,252 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:47,254 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:47,254 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/C is initiating minor compaction (all files) 2024-11-13T18:14:47,254 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/C in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:47,254 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/bf5f1562d53c47e9a937950e82f7d53b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9a1e6b7310394471a4e196b40d8f3a68, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e21a1d784d0b4df09e2ecabbf4a23191] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=35.2 K 2024-11-13T18:14:47,256 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5f1562d53c47e9a937950e82f7d53b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731521684441 2024-11-13T18:14:47,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742503_1679 (size=4469) 2024-11-13T18:14:47,258 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a1e6b7310394471a4e196b40d8f3a68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731521684450 2024-11-13T18:14:47,259 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e21a1d784d0b4df09e2ecabbf4a23191, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731521685101 2024-11-13T18:14:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:47,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:47,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742504_1680 (size=12154) 2024-11-13T18:14:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521747302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521747302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521747305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111395246000f0f643898803e8d2a856f1c5_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111395246000f0f643898803e8d2a856f1c5_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:47,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/2265b3dec0b24a08bf158b8de0e623b4, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:47,315 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#C#compaction#584 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:47,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/2265b3dec0b24a08bf158b8de0e623b4 is 175, key is test_row_0/A:col10/1731521686687/Put/seqid=0 2024-11-13T18:14:47,315 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/091fcc07ab45498aac74feb00f767b0a is 50, key is test_row_0/C:col10/1731521685105/Put/seqid=0 2024-11-13T18:14:47,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521747324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521747324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742506_1682 (size=12104) 2024-11-13T18:14:47,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742505_1681 (size=30955) 2024-11-13T18:14:47,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521747415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521747417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521747425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521747624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521747624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521747644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,663 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#A#compaction#581 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:47,664 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/c0066bc222444f1eb9bc42b8364a9d46 is 175, key is test_row_0/A:col10/1731521685105/Put/seqid=0 2024-11-13T18:14:47,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742507_1683 (size=31058) 2024-11-13T18:14:47,686 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/c0066bc222444f1eb9bc42b8364a9d46 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/c0066bc222444f1eb9bc42b8364a9d46 2024-11-13T18:14:47,697 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/A of a5a3ed238deaa2c83ee0dd5f472061c5 into c0066bc222444f1eb9bc42b8364a9d46(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:47,697 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:47,697 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/A, priority=13, startTime=1731521687185; duration=0sec 2024-11-13T18:14:47,697 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:47,697 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:14:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:14:47,751 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/2265b3dec0b24a08bf158b8de0e623b4 2024-11-13T18:14:47,753 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/091fcc07ab45498aac74feb00f767b0a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/091fcc07ab45498aac74feb00f767b0a 2024-11-13T18:14:47,760 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/C of a5a3ed238deaa2c83ee0dd5f472061c5 into 091fcc07ab45498aac74feb00f767b0a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:47,760 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:47,760 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/C, priority=13, startTime=1731521687185; duration=0sec 2024-11-13T18:14:47,760 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:47,760 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:14:47,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/41622dd2a170429fbca70c4e182c6010 is 50, key is test_row_0/B:col10/1731521686687/Put/seqid=0 2024-11-13T18:14:47,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742508_1684 (size=12001) 2024-11-13T18:14:47,775 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/41622dd2a170429fbca70c4e182c6010 2024-11-13T18:14:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/ecc85ac0b89f451db9ee53656e56dbe1 is 50, key is test_row_0/C:col10/1731521686687/Put/seqid=0 2024-11-13T18:14:47,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742509_1685 (size=12001) 2024-11-13T18:14:47,788 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/ecc85ac0b89f451db9ee53656e56dbe1 2024-11-13T18:14:47,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/2265b3dec0b24a08bf158b8de0e623b4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2265b3dec0b24a08bf158b8de0e623b4 2024-11-13T18:14:47,797 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2265b3dec0b24a08bf158b8de0e623b4, entries=150, sequenceid=69, filesize=30.2 K 2024-11-13T18:14:47,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/41622dd2a170429fbca70c4e182c6010 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/41622dd2a170429fbca70c4e182c6010 2024-11-13T18:14:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,804 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/41622dd2a170429fbca70c4e182c6010, entries=150, sequenceid=69, filesize=11.7 K 2024-11-13T18:14:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/ecc85ac0b89f451db9ee53656e56dbe1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ecc85ac0b89f451db9ee53656e56dbe1 2024-11-13T18:14:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,810 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ecc85ac0b89f451db9ee53656e56dbe1, entries=150, sequenceid=69, filesize=11.7 K 2024-11-13T18:14:47,810 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for a5a3ed238deaa2c83ee0dd5f472061c5 in 600ms, sequenceid=69, compaction requested=false 2024-11-13T18:14:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-13T18:14:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-13T18:14:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-13T18:14:47,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2190 sec 2024-11-13T18:14:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.2250 sec 2024-11-13T18:14:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:47,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T18:14:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:47,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:47,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:47,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:47,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:47,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113661a86ef1f7340b9afe240aec9f91bae_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521687296/Put/seqid=0 2024-11-13T18:14:47,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742510_1686 (size=14594) 2024-11-13T18:14:47,892 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:47,896 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113661a86ef1f7340b9afe240aec9f91bae_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113661a86ef1f7340b9afe240aec9f91bae_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:47,897 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/09c46066f78342c193cdfc7308368220, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:47,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/09c46066f78342c193cdfc7308368220 is 175, key is test_row_0/A:col10/1731521687296/Put/seqid=0 2024-11-13T18:14:47,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742511_1687 (size=39549) 2024-11-13T18:14:47,903 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/09c46066f78342c193cdfc7308368220 2024-11-13T18:14:47,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521747904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521747905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/5161fa517b2d44b8a2deabfe77de85c5 is 50, key is test_row_0/B:col10/1731521687296/Put/seqid=0 2024-11-13T18:14:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742512_1688 (size=12001) 2024-11-13T18:14:47,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/5161fa517b2d44b8a2deabfe77de85c5 2024-11-13T18:14:47,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/d4fb4cdc4425475596fbfa19d3edd623 is 50, key is test_row_0/C:col10/1731521687296/Put/seqid=0 2024-11-13T18:14:47,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742513_1689 (size=12001) 2024-11-13T18:14:47,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521747934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521747934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:47,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521747954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521748014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521748014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521748223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521748224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/d4fb4cdc4425475596fbfa19d3edd623 2024-11-13T18:14:48,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/09c46066f78342c193cdfc7308368220 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/09c46066f78342c193cdfc7308368220 2024-11-13T18:14:48,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/09c46066f78342c193cdfc7308368220, entries=200, sequenceid=87, filesize=38.6 K 2024-11-13T18:14:48,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/5161fa517b2d44b8a2deabfe77de85c5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5161fa517b2d44b8a2deabfe77de85c5 2024-11-13T18:14:48,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5161fa517b2d44b8a2deabfe77de85c5, entries=150, sequenceid=87, filesize=11.7 K 2024-11-13T18:14:48,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/d4fb4cdc4425475596fbfa19d3edd623 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/d4fb4cdc4425475596fbfa19d3edd623 2024-11-13T18:14:48,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/d4fb4cdc4425475596fbfa19d3edd623, entries=150, sequenceid=87, filesize=11.7 K 2024-11-13T18:14:48,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for a5a3ed238deaa2c83ee0dd5f472061c5 in 499ms, sequenceid=87, compaction requested=true 2024-11-13T18:14:48,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:48,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:48,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:48,343 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:48,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:48,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:48,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:48,343 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:48,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:48,344 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:48,344 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/A is initiating minor compaction (all files) 2024-11-13T18:14:48,344 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/A in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:48,344 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/c0066bc222444f1eb9bc42b8364a9d46, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2265b3dec0b24a08bf158b8de0e623b4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/09c46066f78342c193cdfc7308368220] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=99.2 K 2024-11-13T18:14:48,344 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:48,344 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/c0066bc222444f1eb9bc42b8364a9d46, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2265b3dec0b24a08bf158b8de0e623b4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/09c46066f78342c193cdfc7308368220] 2024-11-13T18:14:48,345 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0066bc222444f1eb9bc42b8364a9d46, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731521685101 2024-11-13T18:14:48,345 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:48,345 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/B is initiating minor compaction (all files) 2024-11-13T18:14:48,345 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/B in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:48,345 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/305935362a5841c1bdbea62270831183, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/41622dd2a170429fbca70c4e182c6010, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5161fa517b2d44b8a2deabfe77de85c5] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=35.3 K 2024-11-13T18:14:48,345 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2265b3dec0b24a08bf158b8de0e623b4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731521686680 2024-11-13T18:14:48,346 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09c46066f78342c193cdfc7308368220, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731521687296 2024-11-13T18:14:48,346 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 305935362a5841c1bdbea62270831183, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731521685101 2024-11-13T18:14:48,346 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 41622dd2a170429fbca70c4e182c6010, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731521686680 2024-11-13T18:14:48,346 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 5161fa517b2d44b8a2deabfe77de85c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731521687296 2024-11-13T18:14:48,350 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:48,352 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113d7e548f27cac40bb8cef1d7649b3f3dd_a5a3ed238deaa2c83ee0dd5f472061c5 store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:48,354 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#B#compaction#591 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:48,354 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113d7e548f27cac40bb8cef1d7649b3f3dd_a5a3ed238deaa2c83ee0dd5f472061c5, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:48,354 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d7e548f27cac40bb8cef1d7649b3f3dd_a5a3ed238deaa2c83ee0dd5f472061c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:48,355 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/63006c8588c24b228a4e6693584c5e65 is 50, key is test_row_0/B:col10/1731521687296/Put/seqid=0 2024-11-13T18:14:48,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742514_1690 (size=4469) 2024-11-13T18:14:48,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742515_1691 (size=12207) 2024-11-13T18:14:48,363 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#A#compaction#590 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:48,363 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/5eb09c09b0ae49afb4c2ecb72c1c9837 is 175, key is test_row_0/A:col10/1731521687296/Put/seqid=0 2024-11-13T18:14:48,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742516_1692 (size=31161) 2024-11-13T18:14:48,370 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/5eb09c09b0ae49afb4c2ecb72c1c9837 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5eb09c09b0ae49afb4c2ecb72c1c9837 2024-11-13T18:14:48,374 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/A of a5a3ed238deaa2c83ee0dd5f472061c5 into 5eb09c09b0ae49afb4c2ecb72c1c9837(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:48,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:48,374 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/A, priority=13, startTime=1731521688343; duration=0sec 2024-11-13T18:14:48,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:48,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:14:48,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:48,375 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:48,375 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/C is initiating minor compaction (all files) 2024-11-13T18:14:48,375 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/C in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:48,375 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/091fcc07ab45498aac74feb00f767b0a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ecc85ac0b89f451db9ee53656e56dbe1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/d4fb4cdc4425475596fbfa19d3edd623] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=35.3 K 2024-11-13T18:14:48,376 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 091fcc07ab45498aac74feb00f767b0a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731521685101 2024-11-13T18:14:48,376 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecc85ac0b89f451db9ee53656e56dbe1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731521686680 2024-11-13T18:14:48,376 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4fb4cdc4425475596fbfa19d3edd623, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731521687296 2024-11-13T18:14:48,382 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#C#compaction#592 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:48,382 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/dcad1e9aeae74c71be58ad54b1dc409a is 50, key is test_row_0/C:col10/1731521687296/Put/seqid=0 2024-11-13T18:14:48,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742517_1693 (size=12207) 2024-11-13T18:14:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:48,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:14:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:48,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138f773d33e6494a2f8677463f3f7ac8e3_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521688446/Put/seqid=0 2024-11-13T18:14:48,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742518_1694 (size=14594) 2024-11-13T18:14:48,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521748471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521748471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521748472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521748534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521748534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521748584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521748584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521748584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:14:48,748 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:48,748 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-13T18:14:48,751 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:14:48,752 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:48,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:48,769 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/63006c8588c24b228a4e6693584c5e65 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/63006c8588c24b228a4e6693584c5e65 2024-11-13T18:14:48,797 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/B of a5a3ed238deaa2c83ee0dd5f472061c5 into 63006c8588c24b228a4e6693584c5e65(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:48,797 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:48,797 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/B, priority=13, startTime=1731521688343; duration=0sec 2024-11-13T18:14:48,797 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:48,797 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:14:48,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521748794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521748794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521748795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:48,832 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/dcad1e9aeae74c71be58ad54b1dc409a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/dcad1e9aeae74c71be58ad54b1dc409a 2024-11-13T18:14:48,837 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/C of a5a3ed238deaa2c83ee0dd5f472061c5 into dcad1e9aeae74c71be58ad54b1dc409a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:48,837 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:48,837 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/C, priority=13, startTime=1731521688343; duration=0sec 2024-11-13T18:14:48,838 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:48,838 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:14:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:14:48,875 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:48,892 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138f773d33e6494a2f8677463f3f7ac8e3_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138f773d33e6494a2f8677463f3f7ac8e3_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:48,900 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/1c7432aad66d4b7d8e60155e9f0575ae, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:48,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/1c7432aad66d4b7d8e60155e9f0575ae is 175, key is test_row_0/A:col10/1731521688446/Put/seqid=0 2024-11-13T18:14:48,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:14:48,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:48,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:48,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:48,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:48,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:48,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742519_1695 (size=39549) 2024-11-13T18:14:48,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/1c7432aad66d4b7d8e60155e9f0575ae 2024-11-13T18:14:48,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/6d0dc5f45d4e4cf284f1367331b267ba is 50, key is test_row_0/B:col10/1731521688446/Put/seqid=0 2024-11-13T18:14:48,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742520_1696 (size=12001) 2024-11-13T18:14:48,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/6d0dc5f45d4e4cf284f1367331b267ba 2024-11-13T18:14:48,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/ababe02cd80e46959f14ff70b73093b0 is 50, key is test_row_0/C:col10/1731521688446/Put/seqid=0 2024-11-13T18:14:48,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742521_1697 (size=12001) 2024-11-13T18:14:49,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521749044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521749054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:14:49,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:49,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:49,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:49,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:49,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:14:49,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521749104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521749104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521749104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:14:49,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:49,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:49,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:49,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:49,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:49,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:49,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:14:49,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:49,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:49,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:49,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:49,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:14:49,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/ababe02cd80e46959f14ff70b73093b0 2024-11-13T18:14:49,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/1c7432aad66d4b7d8e60155e9f0575ae as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/1c7432aad66d4b7d8e60155e9f0575ae 2024-11-13T18:14:49,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/1c7432aad66d4b7d8e60155e9f0575ae, entries=200, sequenceid=108, filesize=38.6 K 2024-11-13T18:14:49,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/6d0dc5f45d4e4cf284f1367331b267ba as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6d0dc5f45d4e4cf284f1367331b267ba 2024-11-13T18:14:49,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6d0dc5f45d4e4cf284f1367331b267ba, entries=150, sequenceid=108, filesize=11.7 K 2024-11-13T18:14:49,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/ababe02cd80e46959f14ff70b73093b0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ababe02cd80e46959f14ff70b73093b0 2024-11-13T18:14:49,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ababe02cd80e46959f14ff70b73093b0, entries=150, sequenceid=108, filesize=11.7 K 2024-11-13T18:14:49,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a5a3ed238deaa2c83ee0dd5f472061c5 in 973ms, sequenceid=108, compaction requested=false 2024-11-13T18:14:49,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:49,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:14:49,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:49,525 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:14:49,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:49,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:49,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:49,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:49,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:49,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:49,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111387bb72a00f6b4efc94bab592d849c677_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521688457/Put/seqid=0 2024-11-13T18:14:49,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742522_1698 (size=12154) 2024-11-13T18:14:49,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:49,555 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111387bb72a00f6b4efc94bab592d849c677_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111387bb72a00f6b4efc94bab592d849c677_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/daa36fa6ec8049cfae90f5923e5e63e6, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/daa36fa6ec8049cfae90f5923e5e63e6 is 175, key is test_row_0/A:col10/1731521688457/Put/seqid=0 2024-11-13T18:14:49,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742523_1699 (size=30955) 2024-11-13T18:14:49,596 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/daa36fa6ec8049cfae90f5923e5e63e6 2024-11-13T18:14:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/5470759697f74a3f94fd36c745643875 is 50, key is test_row_0/B:col10/1731521688457/Put/seqid=0 2024-11-13T18:14:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:49,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:49,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742524_1700 (size=12001) 2024-11-13T18:14:49,641 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/5470759697f74a3f94fd36c745643875 2024-11-13T18:14:49,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/18d7d65e6e304a0f83a118ef5f8690fc is 50, key is test_row_0/C:col10/1731521688457/Put/seqid=0 2024-11-13T18:14:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521749652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742525_1701 (size=12001) 2024-11-13T18:14:49,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521749659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521749659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521749764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521749774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521749774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:14:49,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521749974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521749984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521749984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:50,065 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/18d7d65e6e304a0f83a118ef5f8690fc 2024-11-13T18:14:50,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521750064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:50,067 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_1' in a5a3ed238deaa2c83ee0dd5f472061c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:50,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/daa36fa6ec8049cfae90f5923e5e63e6 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/daa36fa6ec8049cfae90f5923e5e63e6 2024-11-13T18:14:50,073 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/daa36fa6ec8049cfae90f5923e5e63e6, entries=150, sequenceid=122, filesize=30.2 K 2024-11-13T18:14:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/5470759697f74a3f94fd36c745643875 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5470759697f74a3f94fd36c745643875 2024-11-13T18:14:50,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521750074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:50,076 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in a5a3ed238deaa2c83ee0dd5f472061c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:50,078 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5470759697f74a3f94fd36c745643875, entries=150, sequenceid=122, filesize=11.7 K 2024-11-13T18:14:50,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/18d7d65e6e304a0f83a118ef5f8690fc as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/18d7d65e6e304a0f83a118ef5f8690fc 2024-11-13T18:14:50,082 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/18d7d65e6e304a0f83a118ef5f8690fc, entries=150, sequenceid=122, filesize=11.7 K 2024-11-13T18:14:50,083 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a5a3ed238deaa2c83ee0dd5f472061c5 in 558ms, sequenceid=122, compaction requested=true 2024-11-13T18:14:50,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:50,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:50,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-13T18:14:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-13T18:14:50,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-13T18:14:50,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3320 sec 2024-11-13T18:14:50,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.3380 sec 2024-11-13T18:14:50,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:50,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521689642, max timestamp=1731521690287Num uniques 859; 2024-11-13T18:14:50,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:50,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521689642, max timestamp=1731521690287Num uniques 859; 2024-11-13T18:14:50,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:50,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521689642, max timestamp=1731521690287Num uniques 859; 2024-11-13T18:14:50,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:50,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-13T18:14:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:50,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:50,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:50,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:50,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:50,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:50,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111378ad6d60a96b401485c8728b609b7d54_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521689657/Put/seqid=0 2024-11-13T18:14:50,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742526_1702 (size=14794) 2024-11-13T18:14:50,319 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:50,333 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111378ad6d60a96b401485c8728b609b7d54_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111378ad6d60a96b401485c8728b609b7d54_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:50,335 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/87796796f15347dd847118c870a45be5, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:50,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/87796796f15347dd847118c870a45be5 is 175, key is test_row_0/A:col10/1731521689657/Put/seqid=0 2024-11-13T18:14:50,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742527_1703 (size=39749) 2024-11-13T18:14:50,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521690339, max timestamp=1731521690369Num uniques 859; 2024-11-13T18:14:50,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:50,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521690339, max timestamp=1731521690369Num uniques 859; 2024-11-13T18:14:50,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:50,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521690339, max timestamp=1731521690369Num uniques 859; 2024-11-13T18:14:50,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:50,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521690473, max timestamp=1731521690494Num uniques 1018; 2024-11-13T18:14:50,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:14:50,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521690473, max timestamp=1731521690494Num uniques 1018; 2024-11-13T18:14:50,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:14:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731521690473, max timestamp=1731521690494Num uniques 1018; 2024-11-13T18:14:50,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-13T18:14:50,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731521690554, max timestamp=1731521690571Num uniques 1027; 2024-11-13T18:14:50,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731521690554, max timestamp=1731521690571Num uniques 1009; 2024-11-13T18:14:50,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T18:14:50,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-13T18:14:50,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731521690554, max timestamp=1731521690571Num uniques 1027; 2024-11-13T18:14:50,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T18:14:50,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731521690618, max timestamp=1731521690633Num uniques 1086; 2024-11-13T18:14:50,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731521690618, max timestamp=1731521690633Num uniques 1086; 2024-11-13T18:14:50,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-13T18:14:50,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-13T18:14:50,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731521690618, max timestamp=1731521690633Num uniques 1086; 2024-11-13T18:14:50,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-13T18:14:50,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731521690663, max timestamp=1731521690685Num uniques 1095; 2024-11-13T18:14:50,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-13T18:14:50,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731521690663, max timestamp=1731521690685Num uniques 1095; 2024-11-13T18:14:50,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-11-13T18:14:50,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731521690663, max timestamp=1731521690685Num uniques 1095; 2024-11-13T18:14:50,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-11-13T18:14:50,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731521690725, max timestamp=1731521690748Num uniques 1054; 2024-11-13T18:14:50,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-13T18:14:50,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731521690725, max timestamp=1731521690748Num uniques 1009; 2024-11-13T18:14:50,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=7050, suffix heap size=132904,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=154 2024-11-13T18:14:50,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731521690725, max timestamp=1731521690748Num uniques 1009; 2024-11-13T18:14:50,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=7050, suffix heap size=132904,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=154 2024-11-13T18:14:50,760 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/87796796f15347dd847118c870a45be5 2024-11-13T18:14:50,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/c05b854a787d419ebb3e1181141f3d56 is 50, key is test_row_0/B:col10/1731521689657/Put/seqid=0 2024-11-13T18:14:50,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731521690787, max timestamp=1731521690809Num uniques 1063; 2024-11-13T18:14:50,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731521690787, max timestamp=1731521690809Num uniques 1013; 2024-11-13T18:14:50,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-13T18:14:50,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-13T18:14:50,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731521690787, max timestamp=1731521690809Num uniques 1013; 2024-11-13T18:14:50,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-13T18:14:50,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742528_1704 (size=12151) 2024-11-13T18:14:50,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731521690850, max timestamp=1731521690884Num uniques 1022; 2024-11-13T18:14:50,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-13T18:14:50,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731521690850, max timestamp=1731521690884Num uniques 1022; 2024-11-13T18:14:50,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-13T18:14:50,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731521690850, max timestamp=1731521690884Num uniques 1022; 2024-11-13T18:14:50,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=6870, suffix heap size=134256,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=150 2024-11-13T18:14:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:14:50,906 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:50,907 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-13T18:14:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:14:50,910 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:50,911 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:50,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:50,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731521690926, max timestamp=1731521690960Num uniques 1031; 2024-11-13T18:14:50,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731521690926, max timestamp=1731521690960Num uniques 1031; 2024-11-13T18:14:50,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-13T18:14:50,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-13T18:14:50,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731521690926, max timestamp=1731521690960Num uniques 1009; 2024-11-13T18:14:50,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-13T18:14:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:14:51,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731521691013, max timestamp=1731521691027Num uniques 1140; 2024-11-13T18:14:51,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52200, new segment data size=6870, suffix heap size=146528,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=150 2024-11-13T18:14:51,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731521691013, max timestamp=1731521691027Num uniques 1140; 2024-11-13T18:14:51,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52201, new segment data size=6870, suffix heap size=146528,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=150 2024-11-13T18:14:51,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731521691013, max timestamp=1731521691027Num uniques 1009; 2024-11-13T18:14:51,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46204, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-13T18:14:51,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:14:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731521691050, max timestamp=1731521691091Num uniques 1009; 2024-11-13T18:14:51,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46224, new segment data size=11404, suffix heap size=132904,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=249 2024-11-13T18:14:51,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731521691050, max timestamp=1731521691091Num uniques 1009; 2024-11-13T18:14:51,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46224, new segment data size=11404, suffix heap size=132904,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=249 2024-11-13T18:14:51,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731521691050, max timestamp=1731521691091Num uniques 1009; 2024-11-13T18:14:51,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46223, new segment data size=11404, suffix heap size=132904,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=249 2024-11-13T18:14:51,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731521691172, max timestamp=1731521691184Num uniques 1108; 2024-11-13T18:14:51,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731521691172, max timestamp=1731521691184Num uniques 1108; 2024-11-13T18:14:51,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-13T18:14:51,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-13T18:14:51,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731521691172, max timestamp=1731521691184Num uniques 1108; 2024-11-13T18:14:51,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-13T18:14:51,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:14:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:14:51,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/c05b854a787d419ebb3e1181141f3d56 2024-11-13T18:14:51,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/f1484d8fcd4d44e6a60443600e2f76c8 is 50, key is test_row_0/C:col10/1731521689657/Put/seqid=0 2024-11-13T18:14:51,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731521691221, max timestamp=1731521691238Num uniques 1017; 2024-11-13T18:14:51,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731521691221, max timestamp=1731521691238Num uniques 1017; 2024-11-13T18:14:51,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-13T18:14:51,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-13T18:14:51,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731521691221, max timestamp=1731521691238Num uniques 1017; 2024-11-13T18:14:51,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-13T18:14:51,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742529_1705 (size=12151) 2024-11-13T18:14:51,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731521691251, max timestamp=1731521691280Num uniques 1026; 2024-11-13T18:14:51,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-13T18:14:51,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731521691251, max timestamp=1731521691280Num uniques 1026; 2024-11-13T18:14:51,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731521691251, max timestamp=1731521691280Num uniques 1026; 2024-11-13T18:14:51,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-13T18:14:51,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-13T18:14:51,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731521691305, max timestamp=1731521691335Num uniques 1085; 2024-11-13T18:14:51,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-13T18:14:51,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731521691305, max timestamp=1731521691335Num uniques 1085; 2024-11-13T18:14:51,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-13T18:14:51,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731521691305, max timestamp=1731521691335Num uniques 1085; 2024-11-13T18:14:51,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-13T18:14:51,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731521691364, max timestamp=1731521691380Num uniques 1094; 2024-11-13T18:14:51,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-13T18:14:51,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731521691364, max timestamp=1731521691380Num uniques 1094; 2024-11-13T18:14:51,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-13T18:14:51,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731521691364, max timestamp=1731521691380Num uniques 1044; 2024-11-13T18:14:51,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=13464, suffix heap size=136544,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=294 2024-11-13T18:14:51,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521751429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:51,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521751431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:51,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1731521691415, max timestamp=1731521691435Num uniques 1153; 2024-11-13T18:14:51,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=7005, suffix heap size=147880,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=153 2024-11-13T18:14:51,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1731521691415, max timestamp=1731521691435Num uniques 1153; 2024-11-13T18:14:51,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=7005, suffix heap size=147880,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=153 2024-11-13T18:14:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:51,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1731521691415, max timestamp=1731521691435Num uniques 1153; 2024-11-13T18:14:51,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=7005, suffix heap size=147880,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=153 2024-11-13T18:14:51,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:14:51,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731521691522, max timestamp=1731521691553Num uniques 1012; 2024-11-13T18:14:51,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-11-13T18:14:51,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731521691522, max timestamp=1731521691553Num uniques 1012; 2024-11-13T18:14:51,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-11-13T18:14:51,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731521691522, max timestamp=1731521691553Num uniques 1012; 2024-11-13T18:14:51,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-11-13T18:14:51,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1731521691589, max timestamp=1731521691607Num uniques 1021; 2024-11-13T18:14:51,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-11-13T18:14:51,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7825, totalHeapSize=18024, min timestamp=1731521691589, max timestamp=1731521691607Num uniques 1021; 2024-11-13T18:14:51,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46755, new segment data size=7825, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-11-13T18:14:51,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7820, totalHeapSize=18024, min timestamp=1731521691589, max timestamp=1731521691607Num uniques 1021; 2024-11-13T18:14:51,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46750, new segment data size=7820, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-11-13T18:14:51,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/f1484d8fcd4d44e6a60443600e2f76c8 2024-11-13T18:14:51,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/87796796f15347dd847118c870a45be5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/87796796f15347dd847118c870a45be5 2024-11-13T18:14:51,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/87796796f15347dd847118c870a45be5, entries=200, sequenceid=143, filesize=38.8 K 2024-11-13T18:14:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/c05b854a787d419ebb3e1181141f3d56 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/c05b854a787d419ebb3e1181141f3d56 2024-11-13T18:14:51,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/c05b854a787d419ebb3e1181141f3d56, entries=150, sequenceid=143, filesize=11.9 K 2024-11-13T18:14:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/f1484d8fcd4d44e6a60443600e2f76c8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/f1484d8fcd4d44e6a60443600e2f76c8 2024-11-13T18:14:51,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/f1484d8fcd4d44e6a60443600e2f76c8, entries=150, sequenceid=143, filesize=11.9 K 2024-11-13T18:14:51,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=134.18 KB/137400 for a5a3ed238deaa2c83ee0dd5f472061c5 in 1366ms, sequenceid=143, compaction requested=true 2024-11-13T18:14:51,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:51,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1731521691631, max timestamp=1731521691660Num uniques 1030; 2024-11-13T18:14:51,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1731521691631, max timestamp=1731521691660Num uniques 1030; 2024-11-13T18:14:51,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-11-13T18:14:51,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-11-13T18:14:51,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1731521691631, max timestamp=1731521691660Num uniques 1030; 2024-11-13T18:14:51,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-11-13T18:14:51,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:51,668 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:51,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:51,668 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:51,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:51,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:51,669 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:51,669 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/B is initiating minor compaction (all files) 2024-11-13T18:14:51,669 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/B in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,669 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/63006c8588c24b228a4e6693584c5e65, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6d0dc5f45d4e4cf284f1367331b267ba, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5470759697f74a3f94fd36c745643875, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/c05b854a787d419ebb3e1181141f3d56] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=47.2 K 2024-11-13T18:14:51,670 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 63006c8588c24b228a4e6693584c5e65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731521687296 2024-11-13T18:14:51,670 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:51,670 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/A is initiating minor compaction (all files) 2024-11-13T18:14:51,671 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/A in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,671 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5eb09c09b0ae49afb4c2ecb72c1c9837, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/1c7432aad66d4b7d8e60155e9f0575ae, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/daa36fa6ec8049cfae90f5923e5e63e6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/87796796f15347dd847118c870a45be5] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=138.1 K 2024-11-13T18:14:51,671 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,671 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5eb09c09b0ae49afb4c2ecb72c1c9837, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/1c7432aad66d4b7d8e60155e9f0575ae, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/daa36fa6ec8049cfae90f5923e5e63e6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/87796796f15347dd847118c870a45be5] 2024-11-13T18:14:51,671 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0dc5f45d4e4cf284f1367331b267ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731521687865 2024-11-13T18:14:51,672 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eb09c09b0ae49afb4c2ecb72c1c9837, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731521687296 2024-11-13T18:14:51,672 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 5470759697f74a3f94fd36c745643875, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731521688457 2024-11-13T18:14:51,673 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c7432aad66d4b7d8e60155e9f0575ae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731521687865 2024-11-13T18:14:51,673 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c05b854a787d419ebb3e1181141f3d56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731521689657 2024-11-13T18:14:51,673 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting daa36fa6ec8049cfae90f5923e5e63e6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731521688457 2024-11-13T18:14:51,674 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87796796f15347dd847118c870a45be5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731521689642 2024-11-13T18:14:51,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:51,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:51,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:14:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:51,680 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=67.09 KB heapSize=183.80 KB 2024-11-13T18:14:51,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:51,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:51,683 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#B#compaction#602 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:51,684 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/7154237c81014110bb715eccd819a9ad is 50, key is test_row_0/B:col10/1731521689657/Put/seqid=0 2024-11-13T18:14:51,686 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:51,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742530_1706 (size=12493) 2024-11-13T18:14:51,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b341bcaaa9a54673a4c0c07504864fca_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521691676/Put/seqid=0 2024-11-13T18:14:51,696 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113c65ae886107b4b028e632970250a81e3_a5a3ed238deaa2c83ee0dd5f472061c5 store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:51,698 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113c65ae886107b4b028e632970250a81e3_a5a3ed238deaa2c83ee0dd5f472061c5, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:51,699 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c65ae886107b4b028e632970250a81e3_a5a3ed238deaa2c83ee0dd5f472061c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:51,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742531_1707 (size=12454) 2024-11-13T18:14:51,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742532_1708 (size=4469) 2024-11-13T18:14:51,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521751735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:51,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521751735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:51,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521751741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:51,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521751854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521751854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521751854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:14:52,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521752064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521752064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521752064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521752096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,101 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_1' in a5a3ed238deaa2c83ee0dd5f472061c5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521752106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:52,112 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in a5a3ed238deaa2c83ee0dd5f472061c5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,118 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/7154237c81014110bb715eccd819a9ad as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/7154237c81014110bb715eccd819a9ad 2024-11-13T18:14:52,125 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#A#compaction#603 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:52,125 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/7780983c5a694ad3a2258545fa281c7a is 175, key is test_row_0/A:col10/1731521689657/Put/seqid=0 2024-11-13T18:14:52,125 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b341bcaaa9a54673a4c0c07504864fca_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b341bcaaa9a54673a4c0c07504864fca_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:52,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/6f243e1b92f643d885bbc1a8220e64e8, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:52,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/6f243e1b92f643d885bbc1a8220e64e8 is 175, key is test_row_0/A:col10/1731521691676/Put/seqid=0 2024-11-13T18:14:52,148 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/B of a5a3ed238deaa2c83ee0dd5f472061c5 into 7154237c81014110bb715eccd819a9ad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:52,148 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:52,148 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/B, priority=12, startTime=1731521691668; duration=0sec 2024-11-13T18:14:52,149 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:52,149 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:14:52,149 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:52,152 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:52,152 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/C is initiating minor compaction (all files) 2024-11-13T18:14:52,152 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/C in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:52,152 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/dcad1e9aeae74c71be58ad54b1dc409a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ababe02cd80e46959f14ff70b73093b0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/18d7d65e6e304a0f83a118ef5f8690fc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/f1484d8fcd4d44e6a60443600e2f76c8] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=47.2 K 2024-11-13T18:14:52,157 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting dcad1e9aeae74c71be58ad54b1dc409a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731521687296 2024-11-13T18:14:52,157 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ababe02cd80e46959f14ff70b73093b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731521687865 2024-11-13T18:14:52,157 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 18d7d65e6e304a0f83a118ef5f8690fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731521688457 2024-11-13T18:14:52,157 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting f1484d8fcd4d44e6a60443600e2f76c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731521689657 2024-11-13T18:14:52,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742534_1710 (size=31255) 2024-11-13T18:14:52,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742533_1709 (size=31447) 2024-11-13T18:14:52,172 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=496, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/6f243e1b92f643d885bbc1a8220e64e8 2024-11-13T18:14:52,174 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#C#compaction#605 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:52,174 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/46a48dd3df1f4a6fbc428ad72a89cee9 is 50, key is test_row_0/C:col10/1731521689657/Put/seqid=0 2024-11-13T18:14:52,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/a63919b5ad144b038ee248777464ee10 is 50, key is test_row_0/B:col10/1731521691676/Put/seqid=0 2024-11-13T18:14:52,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742535_1711 (size=12493) 2024-11-13T18:14:52,220 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/46a48dd3df1f4a6fbc428ad72a89cee9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/46a48dd3df1f4a6fbc428ad72a89cee9 2024-11-13T18:14:52,225 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/C of a5a3ed238deaa2c83ee0dd5f472061c5 into 46a48dd3df1f4a6fbc428ad72a89cee9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:52,225 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:52,225 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/C, priority=12, startTime=1731521691677; duration=0sec 2024-11-13T18:14:52,225 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:52,225 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:14:52,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742536_1712 (size=12301) 2024-11-13T18:14:52,229 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/a63919b5ad144b038ee248777464ee10 2024-11-13T18:14:52,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/2832d5976e66473d932fcb47d68cf423 is 50, key is test_row_0/C:col10/1731521691676/Put/seqid=0 2024-11-13T18:14:52,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742537_1713 (size=12301) 2024-11-13T18:14:52,281 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/2832d5976e66473d932fcb47d68cf423 2024-11-13T18:14:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/6f243e1b92f643d885bbc1a8220e64e8 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f243e1b92f643d885bbc1a8220e64e8 2024-11-13T18:14:52,291 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f243e1b92f643d885bbc1a8220e64e8, entries=150, sequenceid=496, filesize=30.5 K 2024-11-13T18:14:52,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/a63919b5ad144b038ee248777464ee10 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/a63919b5ad144b038ee248777464ee10 2024-11-13T18:14:52,299 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/a63919b5ad144b038ee248777464ee10, entries=150, sequenceid=496, filesize=12.0 K 2024-11-13T18:14:52,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/2832d5976e66473d932fcb47d68cf423 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/2832d5976e66473d932fcb47d68cf423 2024-11-13T18:14:52,303 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/2832d5976e66473d932fcb47d68cf423, entries=150, sequenceid=496, filesize=12.0 K 2024-11-13T18:14:52,305 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~183.75 KB/188160, currentSize=114.05 KB/116790 for a5a3ed238deaa2c83ee0dd5f472061c5 in 625ms, sequenceid=496, compaction requested=false 2024-11-13T18:14:52,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:52,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:52,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-13T18:14:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-13T18:14:52,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-13T18:14:52,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-11-13T18:14:52,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4140 sec 2024-11-13T18:14:52,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T18:14:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:52,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113862615869d8f4fa8b08f7d2f659a829f_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521691731/Put/seqid=0 2024-11-13T18:14:52,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742538_1714 (size=12454) 2024-11-13T18:14:52,399 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:52,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521752396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,402 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113862615869d8f4fa8b08f7d2f659a829f_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113862615869d8f4fa8b08f7d2f659a829f_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:52,403 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/717938316ed943509564ecde0fbb6b65, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:52,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/717938316ed943509564ecde0fbb6b65 is 175, key is test_row_0/A:col10/1731521691731/Put/seqid=0 2024-11-13T18:14:52,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521752397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521752399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742539_1715 (size=31255) 2024-11-13T18:14:52,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521752504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521752514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521752514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,576 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/7780983c5a694ad3a2258545fa281c7a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/7780983c5a694ad3a2258545fa281c7a 2024-11-13T18:14:52,580 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/A of a5a3ed238deaa2c83ee0dd5f472061c5 into 7780983c5a694ad3a2258545fa281c7a(size=30.7 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:52,580 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:52,580 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/A, priority=12, startTime=1731521691668; duration=0sec 2024-11-13T18:14:52,580 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:52,580 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:14:52,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521752714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521752724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521752724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:52,808 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=519, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/717938316ed943509564ecde0fbb6b65 2024-11-13T18:14:52,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/0b9e380bf248456c9e09c5910cbb033e is 50, key is test_row_0/B:col10/1731521691731/Put/seqid=0 2024-11-13T18:14:52,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742540_1716 (size=12301) 2024-11-13T18:14:53,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521753024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:53,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521753033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:53,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521753034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:14:53,054 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:53,055 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-13T18:14:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T18:14:53,072 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:53,080 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:53,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T18:14:53,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/0b9e380bf248456c9e09c5910cbb033e 2024-11-13T18:14:53,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/40e7a93301474c6fa4700942553309ff is 50, key is test_row_0/C:col10/1731521691731/Put/seqid=0 2024-11-13T18:14:53,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T18:14:53,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:53,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:53,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:53,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742541_1717 (size=12301) 2024-11-13T18:14:53,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/40e7a93301474c6fa4700942553309ff 2024-11-13T18:14:53,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/717938316ed943509564ecde0fbb6b65 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/717938316ed943509564ecde0fbb6b65 2024-11-13T18:14:53,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/717938316ed943509564ecde0fbb6b65, entries=150, sequenceid=519, filesize=30.5 K 2024-11-13T18:14:53,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/0b9e380bf248456c9e09c5910cbb033e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/0b9e380bf248456c9e09c5910cbb033e 2024-11-13T18:14:53,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/0b9e380bf248456c9e09c5910cbb033e, entries=150, sequenceid=519, filesize=12.0 K 2024-11-13T18:14:53,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/40e7a93301474c6fa4700942553309ff as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/40e7a93301474c6fa4700942553309ff 2024-11-13T18:14:53,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/40e7a93301474c6fa4700942553309ff, entries=150, sequenceid=519, filesize=12.0 K 2024-11-13T18:14:53,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for a5a3ed238deaa2c83ee0dd5f472061c5 in 900ms, sequenceid=519, compaction requested=true 2024-11-13T18:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:53,279 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:53,279 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:53,280 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:53,280 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:53,280 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/B is initiating minor compaction (all files) 2024-11-13T18:14:53,280 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/A is initiating minor compaction (all files) 2024-11-13T18:14:53,280 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/B in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,280 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/A in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,280 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/7154237c81014110bb715eccd819a9ad, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/a63919b5ad144b038ee248777464ee10, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/0b9e380bf248456c9e09c5910cbb033e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=36.2 K 2024-11-13T18:14:53,280 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/7780983c5a694ad3a2258545fa281c7a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f243e1b92f643d885bbc1a8220e64e8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/717938316ed943509564ecde0fbb6b65] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=91.8 K 2024-11-13T18:14:53,280 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,280 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/7780983c5a694ad3a2258545fa281c7a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f243e1b92f643d885bbc1a8220e64e8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/717938316ed943509564ecde0fbb6b65] 2024-11-13T18:14:53,280 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7154237c81014110bb715eccd819a9ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731521689657 2024-11-13T18:14:53,280 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 7780983c5a694ad3a2258545fa281c7a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731521689657 2024-11-13T18:14:53,281 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting a63919b5ad144b038ee248777464ee10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731521691674 2024-11-13T18:14:53,281 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f243e1b92f643d885bbc1a8220e64e8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731521691674 2024-11-13T18:14:53,281 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 717938316ed943509564ecde0fbb6b65, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731521691731 2024-11-13T18:14:53,281 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b9e380bf248456c9e09c5910cbb033e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731521691731 2024-11-13T18:14:53,287 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:53,290 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#B#compaction#612 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:53,290 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/708c9a3101e441a394a4c9121ca20a87 is 50, key is test_row_0/B:col10/1731521691731/Put/seqid=0 2024-11-13T18:14:53,293 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113d210f494687c439684ac739164d6bc6d_a5a3ed238deaa2c83ee0dd5f472061c5 store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:53,295 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113d210f494687c439684ac739164d6bc6d_a5a3ed238deaa2c83ee0dd5f472061c5, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:53,295 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d210f494687c439684ac739164d6bc6d_a5a3ed238deaa2c83ee0dd5f472061c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:53,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742542_1718 (size=12745) 2024-11-13T18:14:53,318 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/708c9a3101e441a394a4c9121ca20a87 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/708c9a3101e441a394a4c9121ca20a87 2024-11-13T18:14:53,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742543_1719 (size=4469) 2024-11-13T18:14:53,323 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/B of a5a3ed238deaa2c83ee0dd5f472061c5 into 708c9a3101e441a394a4c9121ca20a87(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:53,323 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:53,323 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/B, priority=13, startTime=1731521693279; duration=0sec 2024-11-13T18:14:53,323 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:53,323 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:14:53,323 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:53,324 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:53,324 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/C is initiating minor compaction (all files) 2024-11-13T18:14:53,324 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/C in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,324 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/46a48dd3df1f4a6fbc428ad72a89cee9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/2832d5976e66473d932fcb47d68cf423, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/40e7a93301474c6fa4700942553309ff] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=36.2 K 2024-11-13T18:14:53,325 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46a48dd3df1f4a6fbc428ad72a89cee9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731521689657 2024-11-13T18:14:53,325 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2832d5976e66473d932fcb47d68cf423, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731521691674 2024-11-13T18:14:53,326 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40e7a93301474c6fa4700942553309ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731521691731 2024-11-13T18:14:53,332 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#C#compaction#613 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:53,332 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/86abc88bb2154e97a2928961bc97922a is 50, key is test_row_0/C:col10/1731521691731/Put/seqid=0 2024-11-13T18:14:53,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742544_1720 (size=12745) 2024-11-13T18:14:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T18:14:53,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T18:14:53,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111393b436666fd048b6b8ecec8e986ce76f_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521692397/Put/seqid=0 2024-11-13T18:14:53,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742545_1721 (size=12454) 2024-11-13T18:14:53,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,429 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111393b436666fd048b6b8ecec8e986ce76f_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111393b436666fd048b6b8ecec8e986ce76f_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:53,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/8f68ed8c2905403d8ede93542179a5bc, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/8f68ed8c2905403d8ede93542179a5bc is 175, key is test_row_0/A:col10/1731521692397/Put/seqid=0 2024-11-13T18:14:53,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742546_1722 (size=31255) 2024-11-13T18:14:53,447 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=532, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/8f68ed8c2905403d8ede93542179a5bc 2024-11-13T18:14:53,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/13d48b8068a442e18a984e993aa417e9 is 50, key is test_row_0/B:col10/1731521692397/Put/seqid=0 2024-11-13T18:14:53,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742547_1723 (size=12301) 2024-11-13T18:14:53,472 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/13d48b8068a442e18a984e993aa417e9 2024-11-13T18:14:53,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/1920c2763d8f484a80e6828cad9f8212 is 50, key is test_row_0/C:col10/1731521692397/Put/seqid=0 2024-11-13T18:14:53,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742548_1724 (size=12301) 2024-11-13T18:14:53,497 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/1920c2763d8f484a80e6828cad9f8212 2024-11-13T18:14:53,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/8f68ed8c2905403d8ede93542179a5bc as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8f68ed8c2905403d8ede93542179a5bc 2024-11-13T18:14:53,505 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8f68ed8c2905403d8ede93542179a5bc, entries=150, sequenceid=532, filesize=30.5 K 2024-11-13T18:14:53,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/13d48b8068a442e18a984e993aa417e9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/13d48b8068a442e18a984e993aa417e9 2024-11-13T18:14:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,510 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/13d48b8068a442e18a984e993aa417e9, entries=150, sequenceid=532, filesize=12.0 K 2024-11-13T18:14:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/1920c2763d8f484a80e6828cad9f8212 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1920c2763d8f484a80e6828cad9f8212 2024-11-13T18:14:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,517 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1920c2763d8f484a80e6828cad9f8212, entries=150, sequenceid=532, filesize=12.0 K 2024-11-13T18:14:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,518 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for a5a3ed238deaa2c83ee0dd5f472061c5 in 113ms, sequenceid=532, compaction requested=false 2024-11-13T18:14:53,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:53,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-13T18:14:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-13T18:14:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-13T18:14:53,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 438 msec 2024-11-13T18:14:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 467 msec 2024-11-13T18:14:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:53,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:14:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:53,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:53,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:53,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:53,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:53,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136794d77ea9e44cbcac6dd7b2249307b3_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521693585/Put/seqid=0 2024-11-13T18:14:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742549_1725 (size=22618) 2024-11-13T18:14:53,626 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,630 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136794d77ea9e44cbcac6dd7b2249307b3_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136794d77ea9e44cbcac6dd7b2249307b3_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:53,631 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/8d215844107a4a50a52e5c25c5cbb3a1, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:53,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/8d215844107a4a50a52e5c25c5cbb3a1 is 175, key is test_row_0/A:col10/1731521693585/Put/seqid=0 2024-11-13T18:14:53,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742550_1726 (size=66023) 2024-11-13T18:14:53,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=542, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/8d215844107a4a50a52e5c25c5cbb3a1 2024-11-13T18:14:53,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/eee631051a01464982c3c84cc2279737 is 50, key is test_row_0/B:col10/1731521693585/Put/seqid=0 2024-11-13T18:14:53,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521693650, max timestamp=1731521693661Num uniques 859; 2024-11-13T18:14:53,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T18:14:53,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521693650, max timestamp=1731521693661Num uniques 859; 2024-11-13T18:14:53,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:53,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742551_1727 (size=12301) 2024-11-13T18:14:53,686 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:53,686 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-13T18:14:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T18:14:53,691 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:53,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/eee631051a01464982c3c84cc2279737 2024-11-13T18:14:53,692 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:53,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:53,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521753704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:53,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:53,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521753706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:53,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521753707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:53,720 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#A#compaction#611 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:53,720 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/6f10eb33cc354c6987079228fc15425c is 175, key is test_row_0/A:col10/1731521691731/Put/seqid=0 2024-11-13T18:14:53,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/337b0a4fa987481ab3845a7814c00bd9 is 50, key is test_row_0/C:col10/1731521693585/Put/seqid=0 2024-11-13T18:14:53,754 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/86abc88bb2154e97a2928961bc97922a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/86abc88bb2154e97a2928961bc97922a 2024-11-13T18:14:53,761 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/C of a5a3ed238deaa2c83ee0dd5f472061c5 into 86abc88bb2154e97a2928961bc97922a(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:53,762 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:53,762 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/C, priority=13, startTime=1731521693279; duration=0sec 2024-11-13T18:14:53,762 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:53,762 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:14:53,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742552_1728 (size=31699) 2024-11-13T18:14:53,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742553_1729 (size=12301) 2024-11-13T18:14:53,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/337b0a4fa987481ab3845a7814c00bd9 2024-11-13T18:14:53,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/8d215844107a4a50a52e5c25c5cbb3a1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8d215844107a4a50a52e5c25c5cbb3a1 2024-11-13T18:14:53,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8d215844107a4a50a52e5c25c5cbb3a1, entries=350, sequenceid=542, filesize=64.5 K 2024-11-13T18:14:53,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/eee631051a01464982c3c84cc2279737 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/eee631051a01464982c3c84cc2279737 2024-11-13T18:14:53,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/eee631051a01464982c3c84cc2279737, entries=150, sequenceid=542, filesize=12.0 K 2024-11-13T18:14:53,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/337b0a4fa987481ab3845a7814c00bd9 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/337b0a4fa987481ab3845a7814c00bd9 2024-11-13T18:14:53,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/337b0a4fa987481ab3845a7814c00bd9, entries=150, sequenceid=542, filesize=12.0 K 2024-11-13T18:14:53,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for a5a3ed238deaa2c83ee0dd5f472061c5 in 196ms, sequenceid=542, compaction requested=true 2024-11-13T18:14:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:53,785 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-13T18:14:53,786 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:14:53,786 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:14:53,786 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. because compaction request was cancelled 2024-11-13T18:14:53,786 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:14:53,786 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:53,787 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:53,787 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/B is initiating minor compaction (all files) 2024-11-13T18:14:53,787 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/B in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,787 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/708c9a3101e441a394a4c9121ca20a87, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/13d48b8068a442e18a984e993aa417e9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/eee631051a01464982c3c84cc2279737] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=36.5 K 2024-11-13T18:14:53,788 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708c9a3101e441a394a4c9121ca20a87, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731521691731 2024-11-13T18:14:53,788 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13d48b8068a442e18a984e993aa417e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731521692389 2024-11-13T18:14:53,789 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting eee631051a01464982c3c84cc2279737, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1731521693585 2024-11-13T18:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T18:14:53,798 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#B#compaction#620 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:53,799 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/94ff621292c84a4995bfbb14ee0da872 is 50, key is test_row_0/B:col10/1731521693585/Put/seqid=0 2024-11-13T18:14:53,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742554_1730 (size=12847) 2024-11-13T18:14:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:53,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=136.42 KB heapSize=393.47 KB 2024-11-13T18:14:53,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:53,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137955f7ba6ca44b1792e675d6c8068114_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521693702/Put/seqid=0 2024-11-13T18:14:53,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:53,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:53,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742555_1731 (size=14994) 2024-11-13T18:14:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:53,848 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:53,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521753842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:53,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521753844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:53,853 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137955f7ba6ca44b1792e675d6c8068114_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137955f7ba6ca44b1792e675d6c8068114_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521753847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:53,854 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/42b0e66f05f9499fb1e1dbf842d3e208, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:53,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/42b0e66f05f9499fb1e1dbf842d3e208 is 175, key is test_row_0/A:col10/1731521693702/Put/seqid=0 2024-11-13T18:14:53,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742556_1732 (size=39949) 2024-11-13T18:14:53,873 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=575, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/42b0e66f05f9499fb1e1dbf842d3e208 2024-11-13T18:14:53,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/39b092273918412da17eaada3b700e27 is 50, key is test_row_0/B:col10/1731521693702/Put/seqid=0 2024-11-13T18:14:53,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742557_1733 (size=12301) 2024-11-13T18:14:53,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/39b092273918412da17eaada3b700e27 2024-11-13T18:14:53,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/c64705c3a025458d9ebce0c4350f99ce is 50, key is test_row_0/C:col10/1731521693702/Put/seqid=0 2024-11-13T18:14:53,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742558_1734 (size=12301) 2024-11-13T18:14:53,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/c64705c3a025458d9ebce0c4350f99ce 2024-11-13T18:14:53,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/42b0e66f05f9499fb1e1dbf842d3e208 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/42b0e66f05f9499fb1e1dbf842d3e208 2024-11-13T18:14:53,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/42b0e66f05f9499fb1e1dbf842d3e208, entries=200, sequenceid=575, filesize=39.0 K 2024-11-13T18:14:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/39b092273918412da17eaada3b700e27 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/39b092273918412da17eaada3b700e27 2024-11-13T18:14:53,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/39b092273918412da17eaada3b700e27, entries=150, sequenceid=575, filesize=12.0 K 2024-11-13T18:14:53,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/c64705c3a025458d9ebce0c4350f99ce as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/c64705c3a025458d9ebce0c4350f99ce 2024-11-13T18:14:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:53,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/c64705c3a025458d9ebce0c4350f99ce, entries=150, sequenceid=575, filesize=12.0 K 2024-11-13T18:14:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521753957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:53,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~393.42 KB/402864, currentSize=40.25 KB/41220 for a5a3ed238deaa2c83ee0dd5f472061c5 in 142ms, sequenceid=575, compaction requested=true 2024-11-13T18:14:53,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:53,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:53,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:53,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:53,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:14:53,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:53,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T18:14:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:53,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:14:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:53,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130250128e282d4b77860c7b1070ba8650_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521693831/Put/seqid=0 2024-11-13T18:14:54,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:54,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:54,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T18:14:54,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742559_1735 (size=14994) 2024-11-13T18:14:54,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521694020, max timestamp=1731521694053Num uniques 859; 2024-11-13T18:14:54,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:54,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731521694032, max timestamp=1731521694053Num uniques 859; 2024-11-13T18:14:54,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:14:54,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731521694032, max timestamp=1731521694053Num uniques 859; 2024-11-13T18:14:54,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:14:54,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:54,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:54,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521754153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:54,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521754159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:54,174 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/6f10eb33cc354c6987079228fc15425c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f10eb33cc354c6987079228fc15425c 2024-11-13T18:14:54,181 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/A of a5a3ed238deaa2c83ee0dd5f472061c5 into 6f10eb33cc354c6987079228fc15425c(size=31.0 K), total size for store is 165.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:54,181 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:54,181 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/A, priority=13, startTime=1731521693279; duration=0sec 2024-11-13T18:14:54,181 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T18:14:54,181 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:14:54,181 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:14:54,181 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:54,183 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:54,183 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/C is initiating minor compaction (all files) 2024-11-13T18:14:54,183 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/C in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,183 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/86abc88bb2154e97a2928961bc97922a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1920c2763d8f484a80e6828cad9f8212, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/337b0a4fa987481ab3845a7814c00bd9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/c64705c3a025458d9ebce0c4350f99ce] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=48.5 K 2024-11-13T18:14:54,183 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 86abc88bb2154e97a2928961bc97922a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731521691731 2024-11-13T18:14:54,184 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1920c2763d8f484a80e6828cad9f8212, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731521692389 2024-11-13T18:14:54,184 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 337b0a4fa987481ab3845a7814c00bd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1731521693585 2024-11-13T18:14:54,184 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting c64705c3a025458d9ebce0c4350f99ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731521693702 2024-11-13T18:14:54,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521754174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:54,202 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#C#compaction#625 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:54,202 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/9c87762607964bad8d368867c5b93ca2 is 50, key is test_row_0/C:col10/1731521693702/Put/seqid=0 2024-11-13T18:14:54,226 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/94ff621292c84a4995bfbb14ee0da872 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/94ff621292c84a4995bfbb14ee0da872 2024-11-13T18:14:54,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742560_1736 (size=12881) 2024-11-13T18:14:54,232 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/B of a5a3ed238deaa2c83ee0dd5f472061c5 into 94ff621292c84a4995bfbb14ee0da872(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:54,232 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:54,232 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/B, priority=13, startTime=1731521693785; duration=0sec 2024-11-13T18:14:54,232 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:14:54,232 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:14:54,232 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:54,233 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 168926 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:54,234 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/A is initiating minor compaction (all files) 2024-11-13T18:14:54,234 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/A in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,234 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f10eb33cc354c6987079228fc15425c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8f68ed8c2905403d8ede93542179a5bc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8d215844107a4a50a52e5c25c5cbb3a1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/42b0e66f05f9499fb1e1dbf842d3e208] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=165.0 K 2024-11-13T18:14:54,234 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,234 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f10eb33cc354c6987079228fc15425c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8f68ed8c2905403d8ede93542179a5bc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8d215844107a4a50a52e5c25c5cbb3a1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/42b0e66f05f9499fb1e1dbf842d3e208] 2024-11-13T18:14:54,234 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f10eb33cc354c6987079228fc15425c, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731521691731 2024-11-13T18:14:54,235 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f68ed8c2905403d8ede93542179a5bc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731521692389 2024-11-13T18:14:54,235 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d215844107a4a50a52e5c25c5cbb3a1, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1731521693542 2024-11-13T18:14:54,236 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b0e66f05f9499fb1e1dbf842d3e208, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731521693702 2024-11-13T18:14:54,264 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:54,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521754265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:54,277 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411137e54bdd215fd45379d27757e5dc71e0f_a5a3ed238deaa2c83ee0dd5f472061c5 store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:54,280 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411137e54bdd215fd45379d27757e5dc71e0f_a5a3ed238deaa2c83ee0dd5f472061c5, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:54,280 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137e54bdd215fd45379d27757e5dc71e0f_a5a3ed238deaa2c83ee0dd5f472061c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:54,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521754274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:54,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T18:14:54,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742561_1737 (size=4469) 2024-11-13T18:14:54,323 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#A#compaction#626 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:54,323 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/e70be4c91cee410da1ca539d8ad711bf is 175, key is test_row_0/A:col10/1731521693702/Put/seqid=0 2024-11-13T18:14:54,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742562_1738 (size=31835) 2024-11-13T18:14:54,346 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/e70be4c91cee410da1ca539d8ad711bf as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/e70be4c91cee410da1ca539d8ad711bf 2024-11-13T18:14:54,361 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/A of a5a3ed238deaa2c83ee0dd5f472061c5 into e70be4c91cee410da1ca539d8ad711bf(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:54,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:54,362 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/A, priority=12, startTime=1731521693962; duration=0sec 2024-11-13T18:14:54,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:14:54,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:14:54,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:14:54,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-13T18:14:54,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:14:54,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:14:54,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. because compaction request was cancelled 2024-11-13T18:14:54,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:14:54,362 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:14:54,363 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:14:54,363 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:14:54,363 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. because compaction request was cancelled 2024-11-13T18:14:54,363 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:14:54,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:54,438 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130250128e282d4b77860c7b1070ba8650_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130250128e282d4b77860c7b1070ba8650_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:54,439 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/5f1097107c0a4617ba952d81c4af0642, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:54,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/5f1097107c0a4617ba952d81c4af0642 is 175, key is test_row_0/A:col10/1731521693831/Put/seqid=0 2024-11-13T18:14:54,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742563_1739 (size=39949) 2024-11-13T18:14:54,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521754485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521754485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521754494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:54,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:54,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:54,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,631 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/9c87762607964bad8d368867c5b93ca2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9c87762607964bad8d368867c5b93ca2 2024-11-13T18:14:54,636 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/C of a5a3ed238deaa2c83ee0dd5f472061c5 into 9c87762607964bad8d368867c5b93ca2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:54,636 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:54,636 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/C, priority=12, startTime=1731521693962; duration=0sec 2024-11-13T18:14:54,636 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:54,636 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:14:54,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:54,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521754794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:54,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:54,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521754804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:54,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:54,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T18:14:54,844 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=585, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/5f1097107c0a4617ba952d81c4af0642 2024-11-13T18:14:54,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/232c556e1cf64d8c9b75cd4be5e14348 is 50, key is test_row_0/B:col10/1731521693831/Put/seqid=0 2024-11-13T18:14:54,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742564_1740 (size=12301) 2024-11-13T18:14:54,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/232c556e1cf64d8c9b75cd4be5e14348 2024-11-13T18:14:54,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/e2b0786658964be6bd30967cff811bb0 is 50, key is test_row_0/C:col10/1731521693831/Put/seqid=0 2024-11-13T18:14:54,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742565_1741 (size=12301) 2024-11-13T18:14:54,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:54,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:54,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:54,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521755004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:55,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:55,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/e2b0786658964be6bd30967cff811bb0 2024-11-13T18:14:55,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/5f1097107c0a4617ba952d81c4af0642 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5f1097107c0a4617ba952d81c4af0642 2024-11-13T18:14:55,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5f1097107c0a4617ba952d81c4af0642, entries=200, sequenceid=585, filesize=39.0 K 2024-11-13T18:14:55,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/232c556e1cf64d8c9b75cd4be5e14348 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/232c556e1cf64d8c9b75cd4be5e14348 2024-11-13T18:14:55,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/232c556e1cf64d8c9b75cd4be5e14348, entries=150, sequenceid=585, filesize=12.0 K 2024-11-13T18:14:55,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/e2b0786658964be6bd30967cff811bb0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e2b0786658964be6bd30967cff811bb0 2024-11-13T18:14:55,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e2b0786658964be6bd30967cff811bb0, entries=150, sequenceid=585, filesize=12.0 K 2024-11-13T18:14:55,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=135.63 KB/138880 for a5a3ed238deaa2c83ee0dd5f472061c5 in 1312ms, sequenceid=585, compaction requested=true 2024-11-13T18:14:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:55,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:55,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:55,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:55,282 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:14:55,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:55,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:55,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:55,282 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:55,282 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:14:55,282 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:14:55,283 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. because compaction request was cancelled 2024-11-13T18:14:55,283 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:14:55,283 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:14:55,283 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:55,283 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/B is initiating minor compaction (all files) 2024-11-13T18:14:55,283 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/B in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,283 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/94ff621292c84a4995bfbb14ee0da872, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/39b092273918412da17eaada3b700e27, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/232c556e1cf64d8c9b75cd4be5e14348] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=36.6 K 2024-11-13T18:14:55,283 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ff621292c84a4995bfbb14ee0da872, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1731521693585 2024-11-13T18:14:55,284 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:14:55,284 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:14:55,284 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. because compaction request was cancelled 2024-11-13T18:14:55,284 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:14:55,284 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b092273918412da17eaada3b700e27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731521693702 2024-11-13T18:14:55,284 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 232c556e1cf64d8c9b75cd4be5e14348, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731521693831 2024-11-13T18:14:55,291 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#B#compaction#629 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:55,291 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/1379550efd3c4748b37e1c1e8bf4fdb4 is 50, key is test_row_0/B:col10/1731521693831/Put/seqid=0 2024-11-13T18:14:55,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742566_1742 (size=12949) 2024-11-13T18:14:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:55,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=142.33 KB heapSize=408.86 KB 2024-11-13T18:14:55,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:55,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:14:55,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:55,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:14:55,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:55,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:14:55,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111337d89f49a81e43b3a4756a0302fb54b0_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521694149/Put/seqid=0 2024-11-13T18:14:55,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742567_1743 (size=14994) 2024-11-13T18:14:55,323 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:55,327 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111337d89f49a81e43b3a4756a0302fb54b0_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111337d89f49a81e43b3a4756a0302fb54b0_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:55,328 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/f1204133520c410183672c442faacc79, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:55,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/f1204133520c410183672c442faacc79 is 175, key is test_row_0/A:col10/1731521694149/Put/seqid=0 2024-11-13T18:14:55,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742568_1744 (size=39949) 2024-11-13T18:14:55,332 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=627, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/f1204133520c410183672c442faacc79 2024-11-13T18:14:55,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521755332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:55,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:55,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521755334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:55,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/af7b65dfbf334705af1a339dc295658a is 50, key is test_row_0/B:col10/1731521694149/Put/seqid=0 2024-11-13T18:14:55,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742569_1745 (size=12301) 2024-11-13T18:14:55,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/af7b65dfbf334705af1a339dc295658a 2024-11-13T18:14:55,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/4c1d359887404dd4ae40147bb0a77dfc is 50, key is test_row_0/C:col10/1731521694149/Put/seqid=0 2024-11-13T18:14:55,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742570_1746 (size=12301) 2024-11-13T18:14:55,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521755444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:55,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:55,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521755454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:55,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521755654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:55,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521755664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:55,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:55,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:55,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:55,701 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/1379550efd3c4748b37e1c1e8bf4fdb4 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1379550efd3c4748b37e1c1e8bf4fdb4 2024-11-13T18:14:55,706 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/B of a5a3ed238deaa2c83ee0dd5f472061c5 into 1379550efd3c4748b37e1c1e8bf4fdb4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:55,706 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:55,706 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/B, priority=13, startTime=1731521695282; duration=0sec 2024-11-13T18:14:55,706 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:55,706 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:14:55,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/4c1d359887404dd4ae40147bb0a77dfc 2024-11-13T18:14:55,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/f1204133520c410183672c442faacc79 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f1204133520c410183672c442faacc79 2024-11-13T18:14:55,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f1204133520c410183672c442faacc79, entries=200, sequenceid=627, filesize=39.0 K 2024-11-13T18:14:55,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/af7b65dfbf334705af1a339dc295658a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/af7b65dfbf334705af1a339dc295658a 2024-11-13T18:14:55,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/af7b65dfbf334705af1a339dc295658a, entries=150, sequenceid=627, filesize=12.0 K 2024-11-13T18:14:55,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/4c1d359887404dd4ae40147bb0a77dfc as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/4c1d359887404dd4ae40147bb0a77dfc 2024-11-13T18:14:55,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/4c1d359887404dd4ae40147bb0a77dfc, entries=150, sequenceid=627, filesize=12.0 K 2024-11-13T18:14:55,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~142.33 KB/145750, heapSize ~408.81 KB/418624, currentSize=40.25 KB/41220 for a5a3ed238deaa2c83ee0dd5f472061c5 in 521ms, sequenceid=627, compaction requested=true 2024-11-13T18:14:55,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:55,828 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:55,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:14:55,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:14:55,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:14:55,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. because compaction request was cancelled 2024-11-13T18:14:55,828 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:55,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:14:55,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:55,828 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/A is initiating minor compaction (all files) 2024-11-13T18:14:55,828 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/A in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,829 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/e70be4c91cee410da1ca539d8ad711bf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5f1097107c0a4617ba952d81c4af0642, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f1204133520c410183672c442faacc79] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=109.1 K 2024-11-13T18:14:55,829 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,829 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/e70be4c91cee410da1ca539d8ad711bf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5f1097107c0a4617ba952d81c4af0642, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f1204133520c410183672c442faacc79] 2024-11-13T18:14:55,829 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting e70be4c91cee410da1ca539d8ad711bf, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731521693702 2024-11-13T18:14:55,830 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f1097107c0a4617ba952d81c4af0642, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731521693831 2024-11-13T18:14:55,830 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:55,830 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/C is initiating minor compaction (all files) 2024-11-13T18:14:55,830 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1204133520c410183672c442faacc79, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731521694135 2024-11-13T18:14:55,830 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/C in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,830 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9c87762607964bad8d368867c5b93ca2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e2b0786658964be6bd30967cff811bb0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/4c1d359887404dd4ae40147bb0a77dfc] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=36.6 K 2024-11-13T18:14:55,830 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c87762607964bad8d368867c5b93ca2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731521693702 2024-11-13T18:14:55,831 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting e2b0786658964be6bd30967cff811bb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731521693831 2024-11-13T18:14:55,831 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c1d359887404dd4ae40147bb0a77dfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731521694149 2024-11-13T18:14:55,836 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:55,838 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113e9f90e117ae943f8ac1da25c5a51f8ee_a5a3ed238deaa2c83ee0dd5f472061c5 store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:55,841 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113e9f90e117ae943f8ac1da25c5a51f8ee_a5a3ed238deaa2c83ee0dd5f472061c5, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:55,841 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e9f90e117ae943f8ac1da25c5a51f8ee_a5a3ed238deaa2c83ee0dd5f472061c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:55,844 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#C#compaction#634 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T18:14:55,844 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/beb1b1f70d5d48a68a43f80e942b3f47 is 50, key is test_row_0/C:col10/1731521694149/Put/seqid=0 2024-11-13T18:14:55,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:14:55,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:55,851 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T18:14:55,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:55,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:55,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:55,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:55,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:55,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:55,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742571_1747 (size=4469) 2024-11-13T18:14:55,858 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#A#compaction#633 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:55,859 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/01aae5c0421c4662a667b894d3587395 is 175, key is test_row_0/A:col10/1731521694149/Put/seqid=0 2024-11-13T18:14:55,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742572_1748 (size=12983) 2024-11-13T18:14:55,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742573_1749 (size=31937) 2024-11-13T18:14:55,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ca2f9791146f40fcb199742c8dde3c7e_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521695330/Put/seqid=0 2024-11-13T18:14:55,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742574_1750 (size=12454) 2024-11-13T18:14:55,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:55,895 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ca2f9791146f40fcb199742c8dde3c7e_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ca2f9791146f40fcb199742c8dde3c7e_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/2774857fe2ea4406a68a126dcf9cea67, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/2774857fe2ea4406a68a126dcf9cea67 is 175, key is test_row_0/A:col10/1731521695330/Put/seqid=0 2024-11-13T18:14:55,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742575_1751 (size=31255) 2024-11-13T18:14:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:55,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:56,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521696004, max timestamp=1731521696020Num uniques 859; 2024-11-13T18:14:56,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:56,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521696004, max timestamp=1731521696020Num uniques 859; 2024-11-13T18:14:56,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:56,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521696004, max timestamp=1731521696020Num uniques 859; 2024-11-13T18:14:56,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:14:56,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521696049, max timestamp=1731521696068Num uniques 1068; 2024-11-13T18:14:56,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T18:14:56,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521696049, max timestamp=1731521696068Num uniques 1018; 2024-11-13T18:14:56,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:14:56,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731521696049, max timestamp=1731521696068Num uniques 1018; 2024-11-13T18:14:56,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:14:56,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46272 deadline: 1731521756124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521756125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,131 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_1' in a5a3ed238deaa2c83ee0dd5f472061c5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:14:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521756129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731521696117, max timestamp=1731521696129Num uniques 1027; 2024-11-13T18:14:56,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T18:14:56,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731521696117, max timestamp=1731521696129Num uniques 1027; 2024-11-13T18:14:56,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T18:14:56,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731521696117, max timestamp=1731521696129Num uniques 1027; 2024-11-13T18:14:56,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T18:14:56,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521756218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521756219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521756234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521756244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,276 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/beb1b1f70d5d48a68a43f80e942b3f47 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/beb1b1f70d5d48a68a43f80e942b3f47 2024-11-13T18:14:56,287 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/C of a5a3ed238deaa2c83ee0dd5f472061c5 into beb1b1f70d5d48a68a43f80e942b3f47(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:56,287 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:56,287 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/C, priority=13, startTime=1731521695828; duration=0sec 2024-11-13T18:14:56,287 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:56,287 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:14:56,289 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/01aae5c0421c4662a667b894d3587395 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/01aae5c0421c4662a667b894d3587395 2024-11-13T18:14:56,301 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/A of a5a3ed238deaa2c83ee0dd5f472061c5 into 01aae5c0421c4662a667b894d3587395(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:56,301 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:56,301 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/A, priority=13, startTime=1731521695827; duration=0sec 2024-11-13T18:14:56,301 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:56,301 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:14:56,312 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=637, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/2774857fe2ea4406a68a126dcf9cea67 2024-11-13T18:14:56,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521756334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521756334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/aa82e8dc17fa44e6a549be2b6b677304 is 50, key is test_row_0/B:col10/1731521695330/Put/seqid=0 2024-11-13T18:14:56,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742576_1752 (size=12301) 2024-11-13T18:14:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521756445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521756456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521756544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521756544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521756754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521756764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,786 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/aa82e8dc17fa44e6a549be2b6b677304 2024-11-13T18:14:56,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/014474c1701945ea83c6768cb9a14292 is 50, key is test_row_0/C:col10/1731521695330/Put/seqid=0 2024-11-13T18:14:56,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742577_1753 (size=12301) 2024-11-13T18:14:56,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521756854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521756854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,213 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/014474c1701945ea83c6768cb9a14292 2024-11-13T18:14:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/2774857fe2ea4406a68a126dcf9cea67 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2774857fe2ea4406a68a126dcf9cea67 2024-11-13T18:14:57,221 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2774857fe2ea4406a68a126dcf9cea67, entries=150, sequenceid=637, filesize=30.5 K 2024-11-13T18:14:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/aa82e8dc17fa44e6a549be2b6b677304 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/aa82e8dc17fa44e6a549be2b6b677304 2024-11-13T18:14:57,225 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/aa82e8dc17fa44e6a549be2b6b677304, entries=150, sequenceid=637, filesize=12.0 K 2024-11-13T18:14:57,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/014474c1701945ea83c6768cb9a14292 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/014474c1701945ea83c6768cb9a14292 2024-11-13T18:14:57,229 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/014474c1701945ea83c6768cb9a14292, entries=150, sequenceid=637, filesize=12.0 K 2024-11-13T18:14:57,230 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for a5a3ed238deaa2c83ee0dd5f472061c5 in 1379ms, sequenceid=637, compaction requested=true 2024-11-13T18:14:57,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:57,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:57,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-13T18:14:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-13T18:14:57,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-13T18:14:57,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5390 sec 2024-11-13T18:14:57,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.5470 sec 2024-11-13T18:14:57,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731521696201, max timestamp=1731521697266Num uniques 1086; 2024-11-13T18:14:57,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-13T18:14:57,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731521696201, max timestamp=1731521697266Num uniques 1086; 2024-11-13T18:14:57,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-13T18:14:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:57,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=67.09 KB heapSize=182.19 KB 2024-11-13T18:14:57,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:57,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:57,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:57,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:57,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:57,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:14:57,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c5a66b52eb4b41919ec503625aecbd7a_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521696217/Put/seqid=0 2024-11-13T18:14:57,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742578_1754 (size=14994) 2024-11-13T18:14:57,296 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:57,300 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c5a66b52eb4b41919ec503625aecbd7a_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c5a66b52eb4b41919ec503625aecbd7a_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:57,301 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/57623952e7184a598db4678a7a06875e, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:57,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/57623952e7184a598db4678a7a06875e is 175, key is test_row_0/A:col10/1731521696217/Put/seqid=0 2024-11-13T18:14:57,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742579_1755 (size=39949) 2024-11-13T18:14:57,330 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=711, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/57623952e7184a598db4678a7a06875e 2024-11-13T18:14:57,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521697315, max timestamp=1731521697340Num uniques 859; 2024-11-13T18:14:57,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:57,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521697315, max timestamp=1731521697340Num uniques 859; 2024-11-13T18:14:57,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:57,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521697315, max timestamp=1731521697340Num uniques 859; 2024-11-13T18:14:57,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:14:57,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/6619e09d758543be9279c771947c8318 is 50, key is test_row_0/B:col10/1731521696217/Put/seqid=0 2024-11-13T18:14:57,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521757381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742580_1756 (size=12301) 2024-11-13T18:14:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/6619e09d758543be9279c771947c8318 2024-11-13T18:14:57,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521757382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521757384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521757386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/89b49ab0b671466b9eb5f090ce24f42a is 50, key is test_row_0/C:col10/1731521696217/Put/seqid=0 2024-11-13T18:14:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742581_1757 (size=12301) 2024-11-13T18:14:57,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521757494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521757504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521757504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521757504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521757704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521757714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521757714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521757714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:57,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:57,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/89b49ab0b671466b9eb5f090ce24f42a 2024-11-13T18:14:57,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/57623952e7184a598db4678a7a06875e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/57623952e7184a598db4678a7a06875e 2024-11-13T18:14:57,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/57623952e7184a598db4678a7a06875e, entries=200, sequenceid=711, filesize=39.0 K 2024-11-13T18:14:57,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/6619e09d758543be9279c771947c8318 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6619e09d758543be9279c771947c8318 2024-11-13T18:14:57,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6619e09d758543be9279c771947c8318, entries=150, sequenceid=711, filesize=12.0 K 2024-11-13T18:14:57,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/89b49ab0b671466b9eb5f090ce24f42a as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/89b49ab0b671466b9eb5f090ce24f42a 2024-11-13T18:14:57,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/89b49ab0b671466b9eb5f090ce24f42a, entries=150, sequenceid=711, filesize=12.0 K 2024-11-13T18:14:57,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~182.14 KB/186512, currentSize=120.76 KB/123660 for a5a3ed238deaa2c83ee0dd5f472061c5 in 577ms, sequenceid=711, compaction requested=true 2024-11-13T18:14:57,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:57,849 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:57,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:14:57,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:57,849 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:14:57,850 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:57,850 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/A is initiating minor compaction (all files) 2024-11-13T18:14:57,850 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/A in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:57,850 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/01aae5c0421c4662a667b894d3587395, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2774857fe2ea4406a68a126dcf9cea67, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/57623952e7184a598db4678a7a06875e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=100.7 K 2024-11-13T18:14:57,850 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:57,850 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/01aae5c0421c4662a667b894d3587395, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2774857fe2ea4406a68a126dcf9cea67, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/57623952e7184a598db4678a7a06875e] 2024-11-13T18:14:57,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:14:57,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:57,851 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01aae5c0421c4662a667b894d3587395, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731521694149 2024-11-13T18:14:57,851 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:14:57,851 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2774857fe2ea4406a68a126dcf9cea67, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1731521695313 2024-11-13T18:14:57,851 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/B is initiating minor compaction (all files) 2024-11-13T18:14:57,851 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/B in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:57,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:14:57,851 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1379550efd3c4748b37e1c1e8bf4fdb4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/af7b65dfbf334705af1a339dc295658a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/aa82e8dc17fa44e6a549be2b6b677304, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6619e09d758543be9279c771947c8318] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=48.7 K 2024-11-13T18:14:57,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:57,851 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57623952e7184a598db4678a7a06875e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731521696201 2024-11-13T18:14:57,852 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1379550efd3c4748b37e1c1e8bf4fdb4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731521693831 2024-11-13T18:14:57,853 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting af7b65dfbf334705af1a339dc295658a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731521694149 2024-11-13T18:14:57,853 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting aa82e8dc17fa44e6a549be2b6b677304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1731521695313 2024-11-13T18:14:57,854 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 6619e09d758543be9279c771947c8318, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731521696201 2024-11-13T18:14:57,859 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T18:14:57,864 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:14:57,865 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:14:57,866 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113f47b55ad494d40d38c674a6586791a09_a5a3ed238deaa2c83ee0dd5f472061c5 store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:57,869 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113f47b55ad494d40d38c674a6586791a09_a5a3ed238deaa2c83ee0dd5f472061c5, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:57,869 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f47b55ad494d40d38c674a6586791a09_a5a3ed238deaa2c83ee0dd5f472061c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-13T18:14:57,871 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:14:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T18:14:57,872 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:14:57,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:14:57,883 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#B#compaction#642 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:57,883 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/929a2776d4704199b5847f861ddba964 is 50, key is test_row_0/B:col10/1731521696217/Put/seqid=0 2024-11-13T18:14:57,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742582_1758 (size=4469) 2024-11-13T18:14:57,928 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#A#compaction#641 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:57,928 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/25f44cffb3554fa2b6cef359db1f8eee is 175, key is test_row_0/A:col10/1731521696217/Put/seqid=0 2024-11-13T18:14:57,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742583_1759 (size=13085) 2024-11-13T18:14:57,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742584_1760 (size=32039) 2024-11-13T18:14:57,961 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/929a2776d4704199b5847f861ddba964 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/929a2776d4704199b5847f861ddba964 2024-11-13T18:14:57,971 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/B of a5a3ed238deaa2c83ee0dd5f472061c5 into 929a2776d4704199b5847f861ddba964(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:57,971 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:57,971 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/B, priority=12, startTime=1731521697849; duration=0sec 2024-11-13T18:14:57,971 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:14:57,971 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:14:57,971 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:14:57,972 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:14:57,972 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/C is initiating minor compaction (all files) 2024-11-13T18:14:57,972 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/C in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:57,972 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/beb1b1f70d5d48a68a43f80e942b3f47, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/014474c1701945ea83c6768cb9a14292, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/89b49ab0b671466b9eb5f090ce24f42a] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=36.7 K 2024-11-13T18:14:57,973 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting beb1b1f70d5d48a68a43f80e942b3f47, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731521694149 2024-11-13T18:14:57,974 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 014474c1701945ea83c6768cb9a14292, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1731521695313 2024-11-13T18:14:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T18:14:57,975 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 89b49ab0b671466b9eb5f090ce24f42a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731521696201 2024-11-13T18:14:57,996 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#C#compaction#643 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:14:57,996 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/1b0c0655450e42d2ae2f3d52626d84fc is 50, key is test_row_0/C:col10/1731521696217/Put/seqid=0 2024-11-13T18:14:58,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742585_1761 (size=13085) 2024-11-13T18:14:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:58,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T18:14:58,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:58,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:58,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:58,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:58,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:58,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:14:58,024 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/1b0c0655450e42d2ae2f3d52626d84fc as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1b0c0655450e42d2ae2f3d52626d84fc 2024-11-13T18:14:58,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,034 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/C of a5a3ed238deaa2c83ee0dd5f472061c5 into 1b0c0655450e42d2ae2f3d52626d84fc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:58,034 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:58,034 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/C, priority=13, startTime=1731521697851; duration=0sec 2024-11-13T18:14:58,034 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:58,034 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:14:58,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c3c1a066fc0b44859a38fcddce28b571_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521697381/Put/seqid=0 2024-11-13T18:14:58,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521758034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521758038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521758040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521758041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742586_1762 (size=12454) 2024-11-13T18:14:58,055 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:58,058 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c3c1a066fc0b44859a38fcddce28b571_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c3c1a066fc0b44859a38fcddce28b571_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:58,061 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/84361e006b164a749512ff70fec07a4c, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:58,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/84361e006b164a749512ff70fec07a4c is 175, key is test_row_0/A:col10/1731521697381/Put/seqid=0 2024-11-13T18:14:58,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742587_1763 (size=31255) 2024-11-13T18:14:58,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521758148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521758157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521758157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521758161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T18:14:58,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:58,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:58,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521758354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521758366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521758366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521758374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,400 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/25f44cffb3554fa2b6cef359db1f8eee as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/25f44cffb3554fa2b6cef359db1f8eee 2024-11-13T18:14:58,434 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/A of a5a3ed238deaa2c83ee0dd5f472061c5 into 25f44cffb3554fa2b6cef359db1f8eee(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:14:58,434 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:58,434 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/A, priority=13, startTime=1731521697849; duration=0sec 2024-11-13T18:14:58,435 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:14:58,435 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:14:58,489 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=747, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/84361e006b164a749512ff70fec07a4c 2024-11-13T18:14:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T18:14:58,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:58,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:58,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/20c82a96f46f41d69d838668e0ea584d is 50, key is test_row_0/B:col10/1731521697381/Put/seqid=0 2024-11-13T18:14:58,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742588_1764 (size=12301) 2024-11-13T18:14:58,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:58,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:58,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521758666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521758674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521758674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521758684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:58,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:58,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:58,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:58,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T18:14:59,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/20c82a96f46f41d69d838668e0ea584d 2024-11-13T18:14:59,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/a432a147123546b3993c3b31cbb0cfe5 is 50, key is test_row_0/C:col10/1731521697381/Put/seqid=0 2024-11-13T18:14:59,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742589_1765 (size=12301) 2024-11-13T18:14:59,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:59,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:59,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521759184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:59,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:59,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:59,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:59,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:59,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521759195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521759194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:59,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:14:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521759195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:14:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:14:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:14:59,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:59,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:59,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:59,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:59,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:59,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:59,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:14:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:59,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:14:59,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/a432a147123546b3993c3b31cbb0cfe5 2024-11-13T18:14:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/84361e006b164a749512ff70fec07a4c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/84361e006b164a749512ff70fec07a4c 2024-11-13T18:14:59,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/84361e006b164a749512ff70fec07a4c, entries=150, sequenceid=747, filesize=30.5 K 2024-11-13T18:14:59,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/20c82a96f46f41d69d838668e0ea584d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/20c82a96f46f41d69d838668e0ea584d 2024-11-13T18:14:59,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/20c82a96f46f41d69d838668e0ea584d, entries=150, sequenceid=747, filesize=12.0 K 2024-11-13T18:14:59,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/a432a147123546b3993c3b31cbb0cfe5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/a432a147123546b3993c3b31cbb0cfe5 2024-11-13T18:14:59,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/a432a147123546b3993c3b31cbb0cfe5, entries=150, sequenceid=747, filesize=12.0 K 2024-11-13T18:14:59,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for a5a3ed238deaa2c83ee0dd5f472061c5 in 1471ms, sequenceid=747, compaction requested=false 2024-11-13T18:14:59,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:59,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:14:59,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:59,593 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:14:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:14:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:14:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:14:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:14:59,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a706d5d99ea547e2a6c74dd2b56ad7ad_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521698039/Put/seqid=0 2024-11-13T18:14:59,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742590_1766 (size=12454) 2024-11-13T18:14:59,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,655 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a706d5d99ea547e2a6c74dd2b56ad7ad_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a706d5d99ea547e2a6c74dd2b56ad7ad_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:14:59,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/971bad7a1ce445b6854949af0f9fb133, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:14:59,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/971bad7a1ce445b6854949af0f9fb133 is 175, key is test_row_0/A:col10/1731521698039/Put/seqid=0 2024-11-13T18:14:59,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742591_1767 (size=31255) 2024-11-13T18:14:59,687 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=760, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/971bad7a1ce445b6854949af0f9fb133 2024-11-13T18:14:59,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/9c0f497255bb48b2add40f5ea2c416b5 is 50, key is test_row_0/B:col10/1731521698039/Put/seqid=0 2024-11-13T18:14:59,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742592_1768 (size=12301) 2024-11-13T18:14:59,727 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=760 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/9c0f497255bb48b2add40f5ea2c416b5 2024-11-13T18:14:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/1eb25c3143a84311b871885e37ad1583 is 50, key is test_row_0/C:col10/1731521698039/Put/seqid=0 2024-11-13T18:14:59,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742593_1769 (size=12301) 2024-11-13T18:14:59,782 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=760 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/1eb25c3143a84311b871885e37ad1583 2024-11-13T18:14:59,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/971bad7a1ce445b6854949af0f9fb133 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/971bad7a1ce445b6854949af0f9fb133 2024-11-13T18:14:59,796 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/971bad7a1ce445b6854949af0f9fb133, entries=150, sequenceid=760, filesize=30.5 K 2024-11-13T18:14:59,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/9c0f497255bb48b2add40f5ea2c416b5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c0f497255bb48b2add40f5ea2c416b5 2024-11-13T18:14:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,802 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c0f497255bb48b2add40f5ea2c416b5, entries=150, sequenceid=760, filesize=12.0 K 2024-11-13T18:14:59,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/1eb25c3143a84311b871885e37ad1583 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1eb25c3143a84311b871885e37ad1583 2024-11-13T18:14:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,812 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1eb25c3143a84311b871885e37ad1583, entries=150, sequenceid=760, filesize=12.0 K 2024-11-13T18:14:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,813 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for a5a3ed238deaa2c83ee0dd5f472061c5 in 220ms, sequenceid=760, compaction requested=true 2024-11-13T18:14:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:14:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:14:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-13T18:14:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-13T18:14:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-13T18:14:59,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9430 sec 2024-11-13T18:14:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.9530 sec 2024-11-13T18:14:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:14:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T18:15:00,016 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:15:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,016 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:15:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-13T18:15:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T18:15:00,019 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:15:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,019 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:15:00,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:15:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T18:15:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T18:15:00,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:00,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-13T18:15:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-13T18:15:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-13T18:15:00,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 159 msec 2024-11-13T18:15:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 165 msec 2024-11-13T18:15:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:15:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:15:00,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:00,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:15:00,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:00,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:15:00,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111322dadfbdb6c344a29a958d71efaec36b_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521700252/Put/seqid=0 2024-11-13T18:15:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742594_1770 (size=22618) 2024-11-13T18:15:00,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,292 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111322dadfbdb6c344a29a958d71efaec36b_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111322dadfbdb6c344a29a958d71efaec36b_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:00,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/23b999b32a954645add297cfd15e82e1, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:00,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/23b999b32a954645add297cfd15e82e1 is 175, key is test_row_0/A:col10/1731521700252/Put/seqid=0 2024-11-13T18:15:00,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742595_1771 (size=66023) 2024-11-13T18:15:00,302 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=771, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/23b999b32a954645add297cfd15e82e1 2024-11-13T18:15:00,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/88ab810b3e734534aaa7e3c07dda7a7e is 50, key is test_row_0/B:col10/1731521700252/Put/seqid=0 2024-11-13T18:15:00,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742596_1772 (size=12301) 2024-11-13T18:15:00,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=771 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/88ab810b3e734534aaa7e3c07dda7a7e 2024-11-13T18:15:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T18:15:00,334 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:15:00,335 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:15:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-13T18:15:00,336 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:15:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:15:00,337 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:15:00,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:15:00,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731521700285, max timestamp=1731521700337Num uniques 859; 2024-11-13T18:15:00,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731521700285, max timestamp=1731521700337Num uniques 859; 2024-11-13T18:15:00,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:15:00,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:15:00,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731521700325, max timestamp=1731521700337Num uniques 859; 2024-11-13T18:15:00,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:15:00,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/80b4e75e9a9e4ae8a3dd1dc31183d945 is 50, key is test_row_0/C:col10/1731521700252/Put/seqid=0 2024-11-13T18:15:00,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742597_1773 (size=12301) 2024-11-13T18:15:00,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=771 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/80b4e75e9a9e4ae8a3dd1dc31183d945 2024-11-13T18:15:00,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/23b999b32a954645add297cfd15e82e1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/23b999b32a954645add297cfd15e82e1 2024-11-13T18:15:00,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/23b999b32a954645add297cfd15e82e1, entries=350, sequenceid=771, filesize=64.5 K 2024-11-13T18:15:00,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/88ab810b3e734534aaa7e3c07dda7a7e as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/88ab810b3e734534aaa7e3c07dda7a7e 2024-11-13T18:15:00,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/88ab810b3e734534aaa7e3c07dda7a7e, entries=150, sequenceid=771, filesize=12.0 K 2024-11-13T18:15:00,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/80b4e75e9a9e4ae8a3dd1dc31183d945 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/80b4e75e9a9e4ae8a3dd1dc31183d945 2024-11-13T18:15:00,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/80b4e75e9a9e4ae8a3dd1dc31183d945, entries=150, sequenceid=771, filesize=12.0 K 2024-11-13T18:15:00,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=123.00 KB/125950 for a5a3ed238deaa2c83ee0dd5f472061c5 in 124ms, sequenceid=771, compaction requested=true 2024-11-13T18:15:00,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:00,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:15:00,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:00,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:15:00,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:15:00,380 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:15:00,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:15:00,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:15:00,380 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:15:00,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-11-13T18:15:00,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:15:00,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:15:00,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:15:00,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:15:00,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:15:00,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:15:00,382 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160572 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:15:00,382 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:15:00,382 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/A is initiating minor compaction (all files) 2024-11-13T18:15:00,382 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/B is initiating minor compaction (all files) 2024-11-13T18:15:00,382 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/A in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,382 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/B in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,382 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/25f44cffb3554fa2b6cef359db1f8eee, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/84361e006b164a749512ff70fec07a4c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/971bad7a1ce445b6854949af0f9fb133, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/23b999b32a954645add297cfd15e82e1] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=156.8 K 2024-11-13T18:15:00,382 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/929a2776d4704199b5847f861ddba964, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/20c82a96f46f41d69d838668e0ea584d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c0f497255bb48b2add40f5ea2c416b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/88ab810b3e734534aaa7e3c07dda7a7e] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=48.8 K 2024-11-13T18:15:00,382 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,382 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/25f44cffb3554fa2b6cef359db1f8eee, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/84361e006b164a749512ff70fec07a4c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/971bad7a1ce445b6854949af0f9fb133, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/23b999b32a954645add297cfd15e82e1] 2024-11-13T18:15:00,383 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25f44cffb3554fa2b6cef359db1f8eee, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731521696201 2024-11-13T18:15:00,383 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 929a2776d4704199b5847f861ddba964, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731521696201 2024-11-13T18:15:00,383 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84361e006b164a749512ff70fec07a4c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731521697381 2024-11-13T18:15:00,383 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c82a96f46f41d69d838668e0ea584d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731521697381 2024-11-13T18:15:00,383 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 971bad7a1ce445b6854949af0f9fb133, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=760, earliestPutTs=1731521698027 2024-11-13T18:15:00,383 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c0f497255bb48b2add40f5ea2c416b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=760, earliestPutTs=1731521698027 2024-11-13T18:15:00,384 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23b999b32a954645add297cfd15e82e1, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=771, earliestPutTs=1731521700236 2024-11-13T18:15:00,384 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 88ab810b3e734534aaa7e3c07dda7a7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=771, earliestPutTs=1731521700239 2024-11-13T18:15:00,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f407c43fd59047ae99bfc2e21a2ae06e_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521700380/Put/seqid=0 2024-11-13T18:15:00,403 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#B#compaction#654 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:15:00,404 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/054387aa4fe54d089708df09351842a5 is 50, key is test_row_0/B:col10/1731521700252/Put/seqid=0 2024-11-13T18:15:00,409 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:00,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521760405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521760405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521760409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521760410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:15:00,449 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113fb85678732934b80b702d29addcb24e9_a5a3ed238deaa2c83ee0dd5f472061c5 store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:00,452 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113fb85678732934b80b702d29addcb24e9_a5a3ed238deaa2c83ee0dd5f472061c5, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:00,452 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fb85678732934b80b702d29addcb24e9_a5a3ed238deaa2c83ee0dd5f472061c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:00,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742598_1774 (size=17534) 2024-11-13T18:15:00,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:00,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:00,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,508 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f407c43fd59047ae99bfc2e21a2ae06e_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f407c43fd59047ae99bfc2e21a2ae06e_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:00,509 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/fe31d73276a94e0d807c42c4ed6258af, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:00,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/fe31d73276a94e0d807c42c4ed6258af is 175, key is test_row_0/A:col10/1731521700380/Put/seqid=0 2024-11-13T18:15:00,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742599_1775 (size=13221) 2024-11-13T18:15:00,522 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/054387aa4fe54d089708df09351842a5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/054387aa4fe54d089708df09351842a5 2024-11-13T18:15:00,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521760515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521760515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,527 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/B of a5a3ed238deaa2c83ee0dd5f472061c5 into 054387aa4fe54d089708df09351842a5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:15:00,527 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:00,527 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/B, priority=12, startTime=1731521700380; duration=0sec 2024-11-13T18:15:00,527 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:15:00,527 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:15:00,527 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:15:00,528 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:15:00,528 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/C is initiating minor compaction (all files) 2024-11-13T18:15:00,528 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/C in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,528 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1b0c0655450e42d2ae2f3d52626d84fc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/a432a147123546b3993c3b31cbb0cfe5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1eb25c3143a84311b871885e37ad1583, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/80b4e75e9a9e4ae8a3dd1dc31183d945] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=48.8 K 2024-11-13T18:15:00,529 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b0c0655450e42d2ae2f3d52626d84fc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731521696201 2024-11-13T18:15:00,529 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting a432a147123546b3993c3b31cbb0cfe5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731521697381 2024-11-13T18:15:00,529 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb25c3143a84311b871885e37ad1583, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=760, earliestPutTs=1731521698027 2024-11-13T18:15:00,529 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 80b4e75e9a9e4ae8a3dd1dc31183d945, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=771, earliestPutTs=1731521700239 2024-11-13T18:15:00,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521760525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521760525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742600_1776 (size=4469) 2024-11-13T18:15:00,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742601_1777 (size=48639) 2024-11-13T18:15:00,550 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=807, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/fe31d73276a94e0d807c42c4ed6258af 2024-11-13T18:15:00,550 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#A#compaction#655 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:15:00,551 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/cd88a222e9854471a37df70086a9c760 is 175, key is test_row_0/A:col10/1731521700252/Put/seqid=0 2024-11-13T18:15:00,555 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#C#compaction#656 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:15:00,556 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/48e38d06382c4ed8a27e2f77491ac7b5 is 50, key is test_row_0/C:col10/1731521700252/Put/seqid=0 2024-11-13T18:15:00,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742602_1778 (size=32175) 2024-11-13T18:15:00,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/9baf725c436a447e97bb7fc7fb84806f is 50, key is test_row_0/B:col10/1731521700380/Put/seqid=0 2024-11-13T18:15:00,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742603_1779 (size=13221) 2024-11-13T18:15:00,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742604_1780 (size=12301) 2024-11-13T18:15:00,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=807 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/9baf725c436a447e97bb7fc7fb84806f 2024-11-13T18:15:00,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/fdf68b4bb117420fb636bad92f37f6c1 is 50, key is test_row_0/C:col10/1731521700380/Put/seqid=0 2024-11-13T18:15:00,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742605_1781 (size=12301) 2024-11-13T18:15:00,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=807 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/fdf68b4bb117420fb636bad92f37f6c1 2024-11-13T18:15:00,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/fe31d73276a94e0d807c42c4ed6258af as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/fe31d73276a94e0d807c42c4ed6258af 2024-11-13T18:15:00,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:00,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:00,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:15:00,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/fe31d73276a94e0d807c42c4ed6258af, entries=250, sequenceid=807, filesize=47.5 K 2024-11-13T18:15:00,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/9baf725c436a447e97bb7fc7fb84806f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9baf725c436a447e97bb7fc7fb84806f 2024-11-13T18:15:00,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9baf725c436a447e97bb7fc7fb84806f, entries=150, sequenceid=807, filesize=12.0 K 2024-11-13T18:15:00,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/fdf68b4bb117420fb636bad92f37f6c1 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/fdf68b4bb117420fb636bad92f37f6c1 2024-11-13T18:15:00,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/fdf68b4bb117420fb636bad92f37f6c1, entries=150, sequenceid=807, filesize=12.0 K 2024-11-13T18:15:00,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for a5a3ed238deaa2c83ee0dd5f472061c5 in 290ms, sequenceid=807, compaction requested=false 2024-11-13T18:15:00,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:00,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:15:00,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:15:00,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:15:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:15:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:00,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ddcdeefd830841cdb4ec4a9b43e01a21_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521700398/Put/seqid=0 2024-11-13T18:15:00,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742606_1782 (size=12454) 2024-11-13T18:15:00,770 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:00,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521760767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521700756, max timestamp=1731521700773Num uniques 859; 2024-11-13T18:15:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521700756, max timestamp=1731521700773Num uniques 859; 2024-11-13T18:15:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:15:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:15:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521700756, max timestamp=1731521700773Num uniques 859; 2024-11-13T18:15:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,780 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ddcdeefd830841cdb4ec4a9b43e01a21_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ddcdeefd830841cdb4ec4a9b43e01a21_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:00,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:15:00,782 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/121799500a2f42c09809f2c217cad5ce, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:00,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/121799500a2f42c09809f2c217cad5ce is 175, key is test_row_0/A:col10/1731521700398/Put/seqid=0 2024-11-13T18:15:00,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:00,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:00,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742607_1783 (size=31255) 2024-11-13T18:15:00,836 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=819, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/121799500a2f42c09809f2c217cad5ce 2024-11-13T18:15:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521760833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521760844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521760844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/51d771b5a79f47379a21136079177c69 is 50, key is test_row_0/B:col10/1731521700398/Put/seqid=0 2024-11-13T18:15:00,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742608_1784 (size=12301) 2024-11-13T18:15:00,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=819 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/51d771b5a79f47379a21136079177c69 2024-11-13T18:15:00,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521760888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/246adc31b6aa45428e420192fa916300 is 50, key is test_row_0/C:col10/1731521700398/Put/seqid=0 2024-11-13T18:15:00,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742609_1785 (size=12301) 2024-11-13T18:15:00,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521760954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521760954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:15:00,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521760964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:00,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:00,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:00,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:00,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:00,978 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/cd88a222e9854471a37df70086a9c760 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/cd88a222e9854471a37df70086a9c760 2024-11-13T18:15:00,983 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/A of a5a3ed238deaa2c83ee0dd5f472061c5 into cd88a222e9854471a37df70086a9c760(size=31.4 K), total size for store is 78.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:15:00,983 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:00,983 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/A, priority=12, startTime=1731521700380; duration=0sec 2024-11-13T18:15:00,983 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:00,983 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:15:00,996 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/48e38d06382c4ed8a27e2f77491ac7b5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/48e38d06382c4ed8a27e2f77491ac7b5 2024-11-13T18:15:01,000 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/C of a5a3ed238deaa2c83ee0dd5f472061c5 into 48e38d06382c4ed8a27e2f77491ac7b5(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:15:01,000 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:01,000 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/C, priority=12, startTime=1731521700380; duration=0sec 2024-11-13T18:15:01,000 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:01,000 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:15:01,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521761103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:01,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:01,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:01,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521761174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521761174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:01,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521761184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:01,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:01,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:01,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=819 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/246adc31b6aa45428e420192fa916300 2024-11-13T18:15:01,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/121799500a2f42c09809f2c217cad5ce as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/121799500a2f42c09809f2c217cad5ce 2024-11-13T18:15:01,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/121799500a2f42c09809f2c217cad5ce, entries=150, sequenceid=819, filesize=30.5 K 2024-11-13T18:15:01,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/51d771b5a79f47379a21136079177c69 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/51d771b5a79f47379a21136079177c69 2024-11-13T18:15:01,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/51d771b5a79f47379a21136079177c69, entries=150, sequenceid=819, filesize=12.0 K 2024-11-13T18:15:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/246adc31b6aa45428e420192fa916300 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/246adc31b6aa45428e420192fa916300 2024-11-13T18:15:01,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/246adc31b6aa45428e420192fa916300, entries=150, sequenceid=819, filesize=12.0 K 2024-11-13T18:15:01,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a5a3ed238deaa2c83ee0dd5f472061c5 in 635ms, sequenceid=819, compaction requested=true 2024-11-13T18:15:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:15:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:01,372 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:15:01,373 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:15:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:15:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:15:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:15:01,373 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112069 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:15:01,373 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/A is initiating minor compaction (all files) 2024-11-13T18:15:01,373 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/A in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,374 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/cd88a222e9854471a37df70086a9c760, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/fe31d73276a94e0d807c42c4ed6258af, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/121799500a2f42c09809f2c217cad5ce] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=109.4 K 2024-11-13T18:15:01,374 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/cd88a222e9854471a37df70086a9c760, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/fe31d73276a94e0d807c42c4ed6258af, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/121799500a2f42c09809f2c217cad5ce] 2024-11-13T18:15:01,374 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:15:01,374 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/B is initiating minor compaction (all files) 2024-11-13T18:15:01,374 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/B in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,374 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/054387aa4fe54d089708df09351842a5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9baf725c436a447e97bb7fc7fb84806f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/51d771b5a79f47379a21136079177c69] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=36.9 K 2024-11-13T18:15:01,374 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd88a222e9854471a37df70086a9c760, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=771, earliestPutTs=1731521700239 2024-11-13T18:15:01,375 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 054387aa4fe54d089708df09351842a5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=771, earliestPutTs=1731521700239 2024-11-13T18:15:01,375 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe31d73276a94e0d807c42c4ed6258af, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=807, earliestPutTs=1731521700368 2024-11-13T18:15:01,375 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 9baf725c436a447e97bb7fc7fb84806f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=807, earliestPutTs=1731521700369 2024-11-13T18:15:01,375 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 121799500a2f42c09809f2c217cad5ce, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=819, earliestPutTs=1731521700395 2024-11-13T18:15:01,376 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 51d771b5a79f47379a21136079177c69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=819, earliestPutTs=1731521700395 2024-11-13T18:15:01,382 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:01,383 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#B#compaction#662 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:15:01,384 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/b9f0525008084b978c83f0c543e83499 is 50, key is test_row_0/B:col10/1731521700398/Put/seqid=0 2024-11-13T18:15:01,386 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113938f781b6f4a42c484bd139bd5b76aec_a5a3ed238deaa2c83ee0dd5f472061c5 store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:01,388 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113938f781b6f4a42c484bd139bd5b76aec_a5a3ed238deaa2c83ee0dd5f472061c5, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:01,388 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113938f781b6f4a42c484bd139bd5b76aec_a5a3ed238deaa2c83ee0dd5f472061c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:01,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742610_1786 (size=13323) 2024-11-13T18:15:01,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T18:15:01,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:15:01,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:15:01,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:15:01,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:15:01,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:15:01,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:15:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:01,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742611_1787 (size=4469) 2024-11-13T18:15:01,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111346b00ca2fa614838bc94b608da3f6c2a_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521700830/Put/seqid=0 2024-11-13T18:15:01,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742612_1788 (size=12454) 2024-11-13T18:15:01,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521761462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:15:01,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521761484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521761484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:01,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521761493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:01,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521761574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:01,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:01,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:01,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:01,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:01,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521761784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:01,820 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/b9f0525008084b978c83f0c543e83499 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/b9f0525008084b978c83f0c543e83499 2024-11-13T18:15:01,822 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#A#compaction#663 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:15:01,823 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/f3989b8775ac45c6a0172b8ebec2a0b5 is 175, key is test_row_0/A:col10/1731521700398/Put/seqid=0 2024-11-13T18:15:01,826 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/B of a5a3ed238deaa2c83ee0dd5f472061c5 into b9f0525008084b978c83f0c543e83499(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:15:01,826 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:01,827 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/B, priority=13, startTime=1731521701372; duration=0sec 2024-11-13T18:15:01,827 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:15:01,827 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:15:01,827 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:15:01,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:15:01,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/C is initiating minor compaction (all files) 2024-11-13T18:15:01,828 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/C in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,828 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/48e38d06382c4ed8a27e2f77491ac7b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/fdf68b4bb117420fb636bad92f37f6c1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/246adc31b6aa45428e420192fa916300] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=36.9 K 2024-11-13T18:15:01,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 48e38d06382c4ed8a27e2f77491ac7b5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=771, earliestPutTs=1731521700239 2024-11-13T18:15:01,829 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting fdf68b4bb117420fb636bad92f37f6c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=807, earliestPutTs=1731521700369 2024-11-13T18:15:01,829 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 246adc31b6aa45428e420192fa916300, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=819, earliestPutTs=1731521700395 2024-11-13T18:15:01,852 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#C#compaction#665 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:15:01,852 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/48f91cf5cf5b412ca7d8fe475776609c is 50, key is test_row_0/C:col10/1731521700398/Put/seqid=0 2024-11-13T18:15:01,855 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:01,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111346b00ca2fa614838bc94b608da3f6c2a_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111346b00ca2fa614838bc94b608da3f6c2a_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:01,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/283fe68df80f4b558303d7feeabe4696, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:01,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/283fe68df80f4b558303d7feeabe4696 is 175, key is test_row_0/A:col10/1731521700830/Put/seqid=0 2024-11-13T18:15:01,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742613_1789 (size=32277) 2024-11-13T18:15:01,882 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/f3989b8775ac45c6a0172b8ebec2a0b5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f3989b8775ac45c6a0172b8ebec2a0b5 2024-11-13T18:15:01,888 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/A of a5a3ed238deaa2c83ee0dd5f472061c5 into f3989b8775ac45c6a0172b8ebec2a0b5(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:15:01,888 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:01,888 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/A, priority=13, startTime=1731521701372; duration=0sec 2024-11-13T18:15:01,888 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:01,888 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:15:01,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:01,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:01,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:01,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:01,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742614_1790 (size=13323) 2024-11-13T18:15:01,910 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/48f91cf5cf5b412ca7d8fe475776609c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/48f91cf5cf5b412ca7d8fe475776609c 2024-11-13T18:15:01,916 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/C of a5a3ed238deaa2c83ee0dd5f472061c5 into 48f91cf5cf5b412ca7d8fe475776609c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:15:01,916 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:01,916 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/C, priority=13, startTime=1731521701373; duration=0sec 2024-11-13T18:15:01,916 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:01,916 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:15:01,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742615_1791 (size=31255) 2024-11-13T18:15:01,927 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=857, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/283fe68df80f4b558303d7feeabe4696 2024-11-13T18:15:01,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/84fb410f5dd7425e941d8ac3cec9b676 is 50, key is test_row_0/B:col10/1731521700830/Put/seqid=0 2024-11-13T18:15:01,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521761994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521761994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:02,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742616_1792 (size=12301) 2024-11-13T18:15:02,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521762004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:02,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:02,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:02,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521762095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:02,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:02,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:02,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:02,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=857 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/84fb410f5dd7425e941d8ac3cec9b676 2024-11-13T18:15:02,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/7c6e73d5715a49d3ba1a2414b3ba0729 is 50, key is test_row_0/C:col10/1731521700830/Put/seqid=0 2024-11-13T18:15:02,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742617_1793 (size=12301) 2024-11-13T18:15:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:15:02,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521762604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:02,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:02,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:02,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:02,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:02,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:02,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=857 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/7c6e73d5715a49d3ba1a2414b3ba0729 2024-11-13T18:15:02,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/283fe68df80f4b558303d7feeabe4696 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/283fe68df80f4b558303d7feeabe4696 2024-11-13T18:15:02,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/283fe68df80f4b558303d7feeabe4696, entries=150, sequenceid=857, filesize=30.5 K 2024-11-13T18:15:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/84fb410f5dd7425e941d8ac3cec9b676 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/84fb410f5dd7425e941d8ac3cec9b676 2024-11-13T18:15:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/84fb410f5dd7425e941d8ac3cec9b676, entries=150, sequenceid=857, filesize=12.0 K 2024-11-13T18:15:02,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/7c6e73d5715a49d3ba1a2414b3ba0729 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/7c6e73d5715a49d3ba1a2414b3ba0729 2024-11-13T18:15:02,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/7c6e73d5715a49d3ba1a2414b3ba0729, entries=150, sequenceid=857, filesize=12.0 K 2024-11-13T18:15:02,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a5a3ed238deaa2c83ee0dd5f472061c5 in 1501ms, sequenceid=857, compaction requested=false 2024-11-13T18:15:02,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:02,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44483 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:15:02,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:02,993 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:15:02,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:15:02,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:02,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:15:02,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:02,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:15:02,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134304b4b148754b6399d1060efd4b6c81_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521701452/Put/seqid=0 2024-11-13T18:15:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. as already flushing 2024-11-13T18:15:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742618_1794 (size=9914) 2024-11-13T18:15:03,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521703083, max timestamp=1731521703110Num uniques 859; 2024-11-13T18:15:03,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521703083, max timestamp=1731521703110Num uniques 859; 2024-11-13T18:15:03,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:15:03,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:15:03,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731521703083, max timestamp=1731521703110Num uniques 859; 2024-11-13T18:15:03,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fda96cf025dc:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:15:03,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521763209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521763209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521763213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521763334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521763334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521763335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:03,489 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134304b4b148754b6399d1060efd4b6c81_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134304b4b148754b6399d1060efd4b6c81_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:03,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/02a31702d90c4c4b9dc845dc86207862, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:03,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/02a31702d90c4c4b9dc845dc86207862 is 175, key is test_row_0/A:col10/1731521701452/Put/seqid=0 2024-11-13T18:15:03,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742619_1795 (size=22561) 2024-11-13T18:15:03,538 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=870, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/02a31702d90c4c4b9dc845dc86207862 2024-11-13T18:15:03,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521763544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521763544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521763556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/680d3cf018044587a23aa2f8a279dea0 is 50, key is test_row_0/B:col10/1731521701452/Put/seqid=0 2024-11-13T18:15:03,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742620_1796 (size=9857) 2024-11-13T18:15:03,629 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=870 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/680d3cf018044587a23aa2f8a279dea0 2024-11-13T18:15:03,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521763624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,635 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in a5a3ed238deaa2c83ee0dd5f472061c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:15:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/ada383c47e8b4a8db299642a3a024076 is 50, key is test_row_0/C:col10/1731521701452/Put/seqid=0 2024-11-13T18:15:03,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742621_1797 (size=9857) 2024-11-13T18:15:03,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521763864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521763864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:03,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521763877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:04,110 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=870 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/ada383c47e8b4a8db299642a3a024076 2024-11-13T18:15:04,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/02a31702d90c4c4b9dc845dc86207862 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/02a31702d90c4c4b9dc845dc86207862 2024-11-13T18:15:04,164 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/02a31702d90c4c4b9dc845dc86207862, entries=100, sequenceid=870, filesize=22.0 K 2024-11-13T18:15:04,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/680d3cf018044587a23aa2f8a279dea0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/680d3cf018044587a23aa2f8a279dea0 2024-11-13T18:15:04,199 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/680d3cf018044587a23aa2f8a279dea0, entries=100, sequenceid=870, filesize=9.6 K 2024-11-13T18:15:04,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/ada383c47e8b4a8db299642a3a024076 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ada383c47e8b4a8db299642a3a024076 2024-11-13T18:15:04,204 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ada383c47e8b4a8db299642a3a024076, entries=100, sequenceid=870, filesize=9.6 K 2024-11-13T18:15:04,205 INFO [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for a5a3ed238deaa2c83ee0dd5f472061c5 in 1212ms, sequenceid=870, compaction requested=true 2024-11-13T18:15:04,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:04,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:04,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-13T18:15:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-13T18:15:04,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-13T18:15:04,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8730 sec 2024-11-13T18:15:04,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.8780 sec 2024-11-13T18:15:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:04,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-13T18:15:04,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:15:04,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:15:04,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:15:04,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:15:04,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:15:04,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:15:04,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521764424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:04,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bd4b225f6a864b78aae244c3e26f0706_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521703207/Put/seqid=0 2024-11-13T18:15:04,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521764426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:04,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:04,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521764427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:04,451 INFO [Thread-2921 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2921. 2024-11-13T18:15:04,451 DEBUG [Thread-2921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:15:04,451 DEBUG [Thread-2921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:04,451 DEBUG [Thread-2921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:04,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:04,461 INFO [Thread-2923 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2923. 2024-11-13T18:15:04,461 DEBUG [Thread-2923 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:15:04,461 DEBUG [Thread-2923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:04,461 DEBUG [Thread-2923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:04,461 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:04,463 INFO [Thread-2919 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2919. 2024-11-13T18:15:04,463 DEBUG [Thread-2919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:15:04,463 DEBUG [Thread-2919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:04,463 DEBUG [Thread-2919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:04,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:04,466 INFO [Thread-2925 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2925. 2024-11-13T18:15:04,466 DEBUG [Thread-2925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:15:04,466 DEBUG [Thread-2925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:04,466 DEBUG [Thread-2925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:04,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:04,468 INFO [Thread-2927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2927. 2024-11-13T18:15:04,468 DEBUG [Thread-2927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:15:04,468 DEBUG [Thread-2927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:04,468 DEBUG [Thread-2927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:04,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:04,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742622_1798 (size=14994) 2024-11-13T18:15:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:15:04,505 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:15:04,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521764544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:04,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:04,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521764554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:04,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521764556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:04,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521764754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:04,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521764764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:04,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521764764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:04,878 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:04,881 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bd4b225f6a864b78aae244c3e26f0706_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bd4b225f6a864b78aae244c3e26f0706_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:04,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/d724034d79704adb935f9e074389d3c5, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:04,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/d724034d79704adb935f9e074389d3c5 is 175, key is test_row_0/A:col10/1731521703207/Put/seqid=0 2024-11-13T18:15:04,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742623_1799 (size=39949) 2024-11-13T18:15:04,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=908, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/d724034d79704adb935f9e074389d3c5 2024-11-13T18:15:04,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/746ac21a71ac4a438dd5038efa8176ea is 50, key is test_row_0/B:col10/1731521703207/Put/seqid=0 2024-11-13T18:15:04,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742624_1800 (size=12301) 2024-11-13T18:15:04,985 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:15:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521765064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:05,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521765074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:05,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521765074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:05,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/746ac21a71ac4a438dd5038efa8176ea 2024-11-13T18:15:05,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/5f905f3df3ec4a9492aeb93aef3ab444 is 50, key is test_row_0/C:col10/1731521703207/Put/seqid=0 2024-11-13T18:15:05,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742625_1801 (size=12301) 2024-11-13T18:15:05,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46218 deadline: 1731521765574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:05,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46298 deadline: 1731521765584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:05,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46282 deadline: 1731521765584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:05,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46244 deadline: 1731521765654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:05,655 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,44483,1731521526892 for 'test_row_0' in a5a3ed238deaa2c83ee0dd5f472061c5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:15:05,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 , the old value is region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:15:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., hostname=fda96cf025dc,44483,1731521526892, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:15:05,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/5f905f3df3ec4a9492aeb93aef3ab444 2024-11-13T18:15:05,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/d724034d79704adb935f9e074389d3c5 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/d724034d79704adb935f9e074389d3c5 2024-11-13T18:15:05,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/d724034d79704adb935f9e074389d3c5, entries=200, sequenceid=908, filesize=39.0 K 2024-11-13T18:15:05,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/746ac21a71ac4a438dd5038efa8176ea as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/746ac21a71ac4a438dd5038efa8176ea 2024-11-13T18:15:05,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/746ac21a71ac4a438dd5038efa8176ea, entries=150, sequenceid=908, filesize=12.0 K 2024-11-13T18:15:05,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/5f905f3df3ec4a9492aeb93aef3ab444 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/5f905f3df3ec4a9492aeb93aef3ab444 2024-11-13T18:15:05,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/5f905f3df3ec4a9492aeb93aef3ab444, entries=150, sequenceid=908, filesize=12.0 K 2024-11-13T18:15:05,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for a5a3ed238deaa2c83ee0dd5f472061c5 in 1327ms, sequenceid=908, compaction requested=true 2024-11-13T18:15:05,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:05,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:15:05,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:05,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:15:05,724 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:15:05,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:05,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5a3ed238deaa2c83ee0dd5f472061c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:15:05,724 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:15:05,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:15:05,725 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 126042 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:15:05,725 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:15:05,725 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/B is initiating minor compaction (all files) 2024-11-13T18:15:05,725 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/A is initiating minor compaction (all files) 2024-11-13T18:15:05,725 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/A in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:05,725 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/B in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:05,725 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/b9f0525008084b978c83f0c543e83499, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/84fb410f5dd7425e941d8ac3cec9b676, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/680d3cf018044587a23aa2f8a279dea0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/746ac21a71ac4a438dd5038efa8176ea] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=46.7 K 2024-11-13T18:15:05,725 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f3989b8775ac45c6a0172b8ebec2a0b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/283fe68df80f4b558303d7feeabe4696, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/02a31702d90c4c4b9dc845dc86207862, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/d724034d79704adb935f9e074389d3c5] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=123.1 K 2024-11-13T18:15:05,725 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:05,725 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. files: [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f3989b8775ac45c6a0172b8ebec2a0b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/283fe68df80f4b558303d7feeabe4696, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/02a31702d90c4c4b9dc845dc86207862, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/d724034d79704adb935f9e074389d3c5] 2024-11-13T18:15:05,726 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting b9f0525008084b978c83f0c543e83499, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=819, earliestPutTs=1731521700395 2024-11-13T18:15:05,726 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3989b8775ac45c6a0172b8ebec2a0b5, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=819, earliestPutTs=1731521700395 2024-11-13T18:15:05,726 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 283fe68df80f4b558303d7feeabe4696, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=857, earliestPutTs=1731521700830 2024-11-13T18:15:05,726 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 84fb410f5dd7425e941d8ac3cec9b676, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=857, earliestPutTs=1731521700830 2024-11-13T18:15:05,726 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 680d3cf018044587a23aa2f8a279dea0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=870, earliestPutTs=1731521701452 2024-11-13T18:15:05,726 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02a31702d90c4c4b9dc845dc86207862, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=870, earliestPutTs=1731521701452 2024-11-13T18:15:05,727 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 746ac21a71ac4a438dd5038efa8176ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1731521703203 2024-11-13T18:15:05,727 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] compactions.Compactor(225): Compacting d724034d79704adb935f9e074389d3c5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1731521703203 2024-11-13T18:15:05,732 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:05,733 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#B#compaction#674 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:15:05,733 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/fff00b22fffd451e93751a22a5a440c0 is 50, key is test_row_0/B:col10/1731521703207/Put/seqid=0 2024-11-13T18:15:05,734 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411130c3be4c4637545e092fd11b5946a1307_a5a3ed238deaa2c83ee0dd5f472061c5 store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:05,744 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411130c3be4c4637545e092fd11b5946a1307_a5a3ed238deaa2c83ee0dd5f472061c5, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:05,744 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130c3be4c4637545e092fd11b5946a1307_a5a3ed238deaa2c83ee0dd5f472061c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:05,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742626_1802 (size=13459) 2024-11-13T18:15:05,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742627_1803 (size=4469) 2024-11-13T18:15:05,761 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#A#compaction#675 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:15:05,762 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/83a79e2dd4b0437bbf5060f227637df2 is 175, key is test_row_0/A:col10/1731521703207/Put/seqid=0 2024-11-13T18:15:05,784 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/fff00b22fffd451e93751a22a5a440c0 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/fff00b22fffd451e93751a22a5a440c0 2024-11-13T18:15:05,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742628_1804 (size=32413) 2024-11-13T18:15:05,798 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/B of a5a3ed238deaa2c83ee0dd5f472061c5 into fff00b22fffd451e93751a22a5a440c0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:15:05,798 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:05,799 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/B, priority=12, startTime=1731521705724; duration=0sec 2024-11-13T18:15:05,799 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:15:05,799 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:B 2024-11-13T18:15:05,799 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:15:05,801 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:15:05,801 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1541): a5a3ed238deaa2c83ee0dd5f472061c5/C is initiating minor compaction (all files) 2024-11-13T18:15:05,801 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5a3ed238deaa2c83ee0dd5f472061c5/C in TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:05,801 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/48f91cf5cf5b412ca7d8fe475776609c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/7c6e73d5715a49d3ba1a2414b3ba0729, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ada383c47e8b4a8db299642a3a024076, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/5f905f3df3ec4a9492aeb93aef3ab444] into tmpdir=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp, totalSize=46.7 K 2024-11-13T18:15:05,801 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 48f91cf5cf5b412ca7d8fe475776609c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=819, earliestPutTs=1731521700395 2024-11-13T18:15:05,802 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c6e73d5715a49d3ba1a2414b3ba0729, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=857, earliestPutTs=1731521700830 2024-11-13T18:15:05,802 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting ada383c47e8b4a8db299642a3a024076, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=870, earliestPutTs=1731521701452 2024-11-13T18:15:05,803 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f905f3df3ec4a9492aeb93aef3ab444, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1731521703203 2024-11-13T18:15:05,805 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/83a79e2dd4b0437bbf5060f227637df2 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/83a79e2dd4b0437bbf5060f227637df2 2024-11-13T18:15:05,810 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/A of a5a3ed238deaa2c83ee0dd5f472061c5 into 83a79e2dd4b0437bbf5060f227637df2(size=31.7 K), total size for store is 31.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:15:05,810 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:05,810 INFO [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/A, priority=12, startTime=1731521705724; duration=0sec 2024-11-13T18:15:05,810 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:05,810 DEBUG [RS:0;fda96cf025dc:44483-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:A 2024-11-13T18:15:05,816 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5a3ed238deaa2c83ee0dd5f472061c5#C#compaction#676 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:15:05,816 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/8807917de086440ba5ab80485e5fc965 is 50, key is test_row_0/C:col10/1731521703207/Put/seqid=0 2024-11-13T18:15:05,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742629_1805 (size=13459) 2024-11-13T18:15:05,824 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/8807917de086440ba5ab80485e5fc965 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/8807917de086440ba5ab80485e5fc965 2024-11-13T18:15:05,828 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5a3ed238deaa2c83ee0dd5f472061c5/C of a5a3ed238deaa2c83ee0dd5f472061c5 into 8807917de086440ba5ab80485e5fc965(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:15:05,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:05,828 INFO [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., storeName=a5a3ed238deaa2c83ee0dd5f472061c5/C, priority=12, startTime=1731521705724; duration=0sec 2024-11-13T18:15:05,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:15:05,828 DEBUG [RS:0;fda96cf025dc:44483-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5a3ed238deaa2c83ee0dd5f472061c5:C 2024-11-13T18:15:06,166 INFO [Thread-2910 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2910. 2024-11-13T18:15:06,166 DEBUG [Thread-2910 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:15:06,166 DEBUG [Thread-2910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:06,166 DEBUG [Thread-2910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:06,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44483 {}] regionserver.HRegion(8855): Flush requested on a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:06,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:15:06,586 INFO [Thread-2916 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2916. 2024-11-13T18:15:06,586 DEBUG [Thread-2916 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:15:06,586 DEBUG [Thread-2916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:06,586 DEBUG [Thread-2916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:06,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:15:06,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:06,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:06,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:15:06,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:06,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:15:06,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:06,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c1284cb8073b4caa9a2007f9cb75ee06_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521706584/Put/seqid=0 2024-11-13T18:15:06,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742630_1806 (size=12454) 2024-11-13T18:15:06,596 INFO [Thread-2914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2914. 2024-11-13T18:15:06,596 DEBUG [Thread-2914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:15:06,596 DEBUG [Thread-2914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:06,596 INFO [Thread-2912 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2912. 2024-11-13T18:15:06,596 DEBUG [Thread-2912 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:15:06,596 DEBUG [Thread-2914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:06,596 DEBUG [Thread-2912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:06,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:06,596 DEBUG [Thread-2912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:06,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:06,994 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:06,997 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c1284cb8073b4caa9a2007f9cb75ee06_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c1284cb8073b4caa9a2007f9cb75ee06_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:06,998 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/bd9e93e0eaf24923a2731f56080cc644, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:06,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/bd9e93e0eaf24923a2731f56080cc644 is 175, key is test_row_0/A:col10/1731521706584/Put/seqid=0 2024-11-13T18:15:07,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742631_1807 (size=31255) 2024-11-13T18:15:07,403 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=921, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/bd9e93e0eaf24923a2731f56080cc644 2024-11-13T18:15:07,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/c8802bf92d914d03add2f44a87cd677d is 50, key is test_row_0/B:col10/1731521706584/Put/seqid=0 2024-11-13T18:15:07,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742632_1808 (size=12301) 2024-11-13T18:15:07,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=921 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/c8802bf92d914d03add2f44a87cd677d 2024-11-13T18:15:07,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/a1b0d3daa07749b6a05cf74df5646418 is 50, key is test_row_0/C:col10/1731521706584/Put/seqid=0 2024-11-13T18:15:07,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742633_1809 (size=12301) 2024-11-13T18:15:07,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=921 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/a1b0d3daa07749b6a05cf74df5646418 2024-11-13T18:15:07,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/bd9e93e0eaf24923a2731f56080cc644 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/bd9e93e0eaf24923a2731f56080cc644 2024-11-13T18:15:07,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/bd9e93e0eaf24923a2731f56080cc644, entries=150, sequenceid=921, filesize=30.5 K 2024-11-13T18:15:07,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/c8802bf92d914d03add2f44a87cd677d as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/c8802bf92d914d03add2f44a87cd677d 2024-11-13T18:15:07,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/c8802bf92d914d03add2f44a87cd677d, entries=150, sequenceid=921, filesize=12.0 K 2024-11-13T18:15:07,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/a1b0d3daa07749b6a05cf74df5646418 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/a1b0d3daa07749b6a05cf74df5646418 2024-11-13T18:15:07,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/a1b0d3daa07749b6a05cf74df5646418, entries=150, sequenceid=921, filesize=12.0 K 2024-11-13T18:15:07,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for a5a3ed238deaa2c83ee0dd5f472061c5 in 1258ms, sequenceid=921, compaction requested=false 2024-11-13T18:15:07,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5a3ed238deaa2c83ee0dd5f472061c5: 2024-11-13T18:15:09,706 INFO [Thread-2908 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2908. 2024-11-13T18:15:09,706 DEBUG [Thread-2908 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:15:09,706 DEBUG [Thread-2908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:09,706 DEBUG [Thread-2908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:09,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:09,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:15:09,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 246 2024-11-13T18:15:09,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-13T18:15:09,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-13T18:15:09,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 242 2024-11-13T18:15:09,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 232 2024-11-13T18:15:09,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:15:09,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2196 2024-11-13T18:15:09,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2188 2024-11-13T18:15:09,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2112 2024-11-13T18:15:09,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2202 2024-11-13T18:15:09,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2186 2024-11-13T18:15:09,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:15:09,707 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:15:09,707 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:15:09,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:09,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:09,707 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:09,708 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:15:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:15:09,711 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521709711"}]},"ts":"1731521709711"} 2024-11-13T18:15:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T18:15:09,713 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:15:09,713 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:15:09,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:15:09,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, UNASSIGN}] 2024-11-13T18:15:09,715 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, UNASSIGN 2024-11-13T18:15:09,716 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=a5a3ed238deaa2c83ee0dd5f472061c5, regionState=CLOSING, regionLocation=fda96cf025dc,44483,1731521526892 2024-11-13T18:15:09,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, UNASSIGN because future has completed 2024-11-13T18:15:09,717 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:15:09,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892}] 2024-11-13T18:15:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T18:15:09,869 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing a5a3ed238deaa2c83ee0dd5f472061c5, disabling compactions & flushes 2024-11-13T18:15:09,870 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. after waiting 0 ms 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:09,870 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing a5a3ed238deaa2c83ee0dd5f472061c5 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=A 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=B 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5a3ed238deaa2c83ee0dd5f472061c5, store=C 2024-11-13T18:15:09,870 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:15:09,876 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113826c69d560524e8aa35dd6fde0eef46d_a5a3ed238deaa2c83ee0dd5f472061c5 is 50, key is test_row_0/A:col10/1731521709705/Put/seqid=0 2024-11-13T18:15:09,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742634_1810 (size=9914) 2024-11-13T18:15:09,890 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:09,901 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113826c69d560524e8aa35dd6fde0eef46d_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113826c69d560524e8aa35dd6fde0eef46d_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:09,902 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/53dcb5dd9b9542f1856cef9d080493ce, store: [table=TestAcidGuarantees family=A region=a5a3ed238deaa2c83ee0dd5f472061c5] 2024-11-13T18:15:09,903 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/53dcb5dd9b9542f1856cef9d080493ce is 175, key is test_row_0/A:col10/1731521709705/Put/seqid=0 2024-11-13T18:15:09,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742635_1811 (size=22561) 2024-11-13T18:15:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T18:15:10,307 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=927, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/53dcb5dd9b9542f1856cef9d080493ce 2024-11-13T18:15:10,312 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/6187e4e4173a4f08858feb9cae1475fa is 50, key is test_row_0/B:col10/1731521709705/Put/seqid=0 2024-11-13T18:15:10,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742636_1812 (size=9857) 2024-11-13T18:15:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T18:15:10,716 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=927 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/6187e4e4173a4f08858feb9cae1475fa 2024-11-13T18:15:10,721 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/87cbb703fa7643a3add059415d950c87 is 50, key is test_row_0/C:col10/1731521709705/Put/seqid=0 2024-11-13T18:15:10,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742637_1813 (size=9857) 2024-11-13T18:15:10,724 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=927 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/87cbb703fa7643a3add059415d950c87 2024-11-13T18:15:10,727 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/A/53dcb5dd9b9542f1856cef9d080493ce as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/53dcb5dd9b9542f1856cef9d080493ce 2024-11-13T18:15:10,730 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/53dcb5dd9b9542f1856cef9d080493ce, entries=100, sequenceid=927, filesize=22.0 K 2024-11-13T18:15:10,730 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/B/6187e4e4173a4f08858feb9cae1475fa as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6187e4e4173a4f08858feb9cae1475fa 2024-11-13T18:15:10,733 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6187e4e4173a4f08858feb9cae1475fa, entries=100, sequenceid=927, filesize=9.6 K 2024-11-13T18:15:10,733 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/.tmp/C/87cbb703fa7643a3add059415d950c87 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/87cbb703fa7643a3add059415d950c87 2024-11-13T18:15:10,736 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/87cbb703fa7643a3add059415d950c87, entries=100, sequenceid=927, filesize=9.6 K 2024-11-13T18:15:10,736 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a5a3ed238deaa2c83ee0dd5f472061c5 in 866ms, sequenceid=927, compaction requested=true 2024-11-13T18:15:10,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/9d0b304ca9ee47e18556111db443884e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/74ee2570a89c445e8de1c1b348d99850, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/0c3522f4a87948bdade2b7c1bd8b3315, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/c0066bc222444f1eb9bc42b8364a9d46, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2265b3dec0b24a08bf158b8de0e623b4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/09c46066f78342c193cdfc7308368220, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5eb09c09b0ae49afb4c2ecb72c1c9837, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/1c7432aad66d4b7d8e60155e9f0575ae, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/daa36fa6ec8049cfae90f5923e5e63e6, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/87796796f15347dd847118c870a45be5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/7780983c5a694ad3a2258545fa281c7a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f243e1b92f643d885bbc1a8220e64e8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f10eb33cc354c6987079228fc15425c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/717938316ed943509564ecde0fbb6b65, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8f68ed8c2905403d8ede93542179a5bc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8d215844107a4a50a52e5c25c5cbb3a1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/42b0e66f05f9499fb1e1dbf842d3e208, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/e70be4c91cee410da1ca539d8ad711bf, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5f1097107c0a4617ba952d81c4af0642, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f1204133520c410183672c442faacc79, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/01aae5c0421c4662a667b894d3587395, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2774857fe2ea4406a68a126dcf9cea67, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/57623952e7184a598db4678a7a06875e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/25f44cffb3554fa2b6cef359db1f8eee, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/84361e006b164a749512ff70fec07a4c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/971bad7a1ce445b6854949af0f9fb133, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/23b999b32a954645add297cfd15e82e1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/cd88a222e9854471a37df70086a9c760, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/fe31d73276a94e0d807c42c4ed6258af, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f3989b8775ac45c6a0172b8ebec2a0b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/121799500a2f42c09809f2c217cad5ce, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/283fe68df80f4b558303d7feeabe4696, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/02a31702d90c4c4b9dc845dc86207862, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/d724034d79704adb935f9e074389d3c5] to archive 2024-11-13T18:15:10,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:15:10,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/9d0b304ca9ee47e18556111db443884e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/9d0b304ca9ee47e18556111db443884e 2024-11-13T18:15:10,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/74ee2570a89c445e8de1c1b348d99850 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/74ee2570a89c445e8de1c1b348d99850 2024-11-13T18:15:10,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/0c3522f4a87948bdade2b7c1bd8b3315 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/0c3522f4a87948bdade2b7c1bd8b3315 2024-11-13T18:15:10,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/c0066bc222444f1eb9bc42b8364a9d46 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/c0066bc222444f1eb9bc42b8364a9d46 2024-11-13T18:15:10,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2265b3dec0b24a08bf158b8de0e623b4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2265b3dec0b24a08bf158b8de0e623b4 2024-11-13T18:15:10,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/09c46066f78342c193cdfc7308368220 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/09c46066f78342c193cdfc7308368220 2024-11-13T18:15:10,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5eb09c09b0ae49afb4c2ecb72c1c9837 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5eb09c09b0ae49afb4c2ecb72c1c9837 2024-11-13T18:15:10,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/1c7432aad66d4b7d8e60155e9f0575ae to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/1c7432aad66d4b7d8e60155e9f0575ae 2024-11-13T18:15:10,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/daa36fa6ec8049cfae90f5923e5e63e6 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/daa36fa6ec8049cfae90f5923e5e63e6 2024-11-13T18:15:10,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/87796796f15347dd847118c870a45be5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/87796796f15347dd847118c870a45be5 2024-11-13T18:15:10,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/7780983c5a694ad3a2258545fa281c7a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/7780983c5a694ad3a2258545fa281c7a 2024-11-13T18:15:10,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f243e1b92f643d885bbc1a8220e64e8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f243e1b92f643d885bbc1a8220e64e8 2024-11-13T18:15:10,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f10eb33cc354c6987079228fc15425c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/6f10eb33cc354c6987079228fc15425c 2024-11-13T18:15:10,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/717938316ed943509564ecde0fbb6b65 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/717938316ed943509564ecde0fbb6b65 2024-11-13T18:15:10,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8f68ed8c2905403d8ede93542179a5bc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8f68ed8c2905403d8ede93542179a5bc 2024-11-13T18:15:10,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8d215844107a4a50a52e5c25c5cbb3a1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/8d215844107a4a50a52e5c25c5cbb3a1 2024-11-13T18:15:10,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/42b0e66f05f9499fb1e1dbf842d3e208 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/42b0e66f05f9499fb1e1dbf842d3e208 2024-11-13T18:15:10,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/e70be4c91cee410da1ca539d8ad711bf to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/e70be4c91cee410da1ca539d8ad711bf 2024-11-13T18:15:10,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5f1097107c0a4617ba952d81c4af0642 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/5f1097107c0a4617ba952d81c4af0642 2024-11-13T18:15:10,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f1204133520c410183672c442faacc79 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f1204133520c410183672c442faacc79 2024-11-13T18:15:10,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/01aae5c0421c4662a667b894d3587395 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/01aae5c0421c4662a667b894d3587395 2024-11-13T18:15:10,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2774857fe2ea4406a68a126dcf9cea67 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/2774857fe2ea4406a68a126dcf9cea67 2024-11-13T18:15:10,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/57623952e7184a598db4678a7a06875e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/57623952e7184a598db4678a7a06875e 2024-11-13T18:15:10,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/25f44cffb3554fa2b6cef359db1f8eee to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/25f44cffb3554fa2b6cef359db1f8eee 2024-11-13T18:15:10,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/84361e006b164a749512ff70fec07a4c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/84361e006b164a749512ff70fec07a4c 2024-11-13T18:15:10,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/971bad7a1ce445b6854949af0f9fb133 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/971bad7a1ce445b6854949af0f9fb133 2024-11-13T18:15:10,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/23b999b32a954645add297cfd15e82e1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/23b999b32a954645add297cfd15e82e1 2024-11-13T18:15:10,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/cd88a222e9854471a37df70086a9c760 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/cd88a222e9854471a37df70086a9c760 2024-11-13T18:15:10,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/fe31d73276a94e0d807c42c4ed6258af to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/fe31d73276a94e0d807c42c4ed6258af 2024-11-13T18:15:10,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f3989b8775ac45c6a0172b8ebec2a0b5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/f3989b8775ac45c6a0172b8ebec2a0b5 2024-11-13T18:15:10,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/121799500a2f42c09809f2c217cad5ce to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/121799500a2f42c09809f2c217cad5ce 2024-11-13T18:15:10,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/283fe68df80f4b558303d7feeabe4696 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/283fe68df80f4b558303d7feeabe4696 2024-11-13T18:15:10,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/02a31702d90c4c4b9dc845dc86207862 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/02a31702d90c4c4b9dc845dc86207862 2024-11-13T18:15:10,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/d724034d79704adb935f9e074389d3c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/d724034d79704adb935f9e074389d3c5 2024-11-13T18:15:10,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1372339871af4324b6063a430896fda5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/4ec5d97f53ec4bf5aae0b4cc4e5a011e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/305935362a5841c1bdbea62270831183, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c9e63ccdbcd474c98022e2364b995e7, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/41622dd2a170429fbca70c4e182c6010, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/63006c8588c24b228a4e6693584c5e65, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5161fa517b2d44b8a2deabfe77de85c5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6d0dc5f45d4e4cf284f1367331b267ba, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5470759697f74a3f94fd36c745643875, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/7154237c81014110bb715eccd819a9ad, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/c05b854a787d419ebb3e1181141f3d56, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/a63919b5ad144b038ee248777464ee10, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/708c9a3101e441a394a4c9121ca20a87, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/0b9e380bf248456c9e09c5910cbb033e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/13d48b8068a442e18a984e993aa417e9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/94ff621292c84a4995bfbb14ee0da872, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/eee631051a01464982c3c84cc2279737, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/39b092273918412da17eaada3b700e27, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1379550efd3c4748b37e1c1e8bf4fdb4, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/232c556e1cf64d8c9b75cd4be5e14348, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/af7b65dfbf334705af1a339dc295658a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/aa82e8dc17fa44e6a549be2b6b677304, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/929a2776d4704199b5847f861ddba964, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6619e09d758543be9279c771947c8318, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/20c82a96f46f41d69d838668e0ea584d, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c0f497255bb48b2add40f5ea2c416b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/054387aa4fe54d089708df09351842a5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/88ab810b3e734534aaa7e3c07dda7a7e, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9baf725c436a447e97bb7fc7fb84806f, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/b9f0525008084b978c83f0c543e83499, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/51d771b5a79f47379a21136079177c69, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/84fb410f5dd7425e941d8ac3cec9b676, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/680d3cf018044587a23aa2f8a279dea0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/746ac21a71ac4a438dd5038efa8176ea] to archive 2024-11-13T18:15:10,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:15:10,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1372339871af4324b6063a430896fda5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1372339871af4324b6063a430896fda5 2024-11-13T18:15:10,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/4ec5d97f53ec4bf5aae0b4cc4e5a011e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/4ec5d97f53ec4bf5aae0b4cc4e5a011e 2024-11-13T18:15:10,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/305935362a5841c1bdbea62270831183 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/305935362a5841c1bdbea62270831183 2024-11-13T18:15:10,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c9e63ccdbcd474c98022e2364b995e7 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c9e63ccdbcd474c98022e2364b995e7 2024-11-13T18:15:10,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/41622dd2a170429fbca70c4e182c6010 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/41622dd2a170429fbca70c4e182c6010 2024-11-13T18:15:10,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/63006c8588c24b228a4e6693584c5e65 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/63006c8588c24b228a4e6693584c5e65 2024-11-13T18:15:10,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5161fa517b2d44b8a2deabfe77de85c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5161fa517b2d44b8a2deabfe77de85c5 2024-11-13T18:15:10,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6d0dc5f45d4e4cf284f1367331b267ba to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6d0dc5f45d4e4cf284f1367331b267ba 2024-11-13T18:15:10,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5470759697f74a3f94fd36c745643875 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/5470759697f74a3f94fd36c745643875 2024-11-13T18:15:10,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/7154237c81014110bb715eccd819a9ad to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/7154237c81014110bb715eccd819a9ad 2024-11-13T18:15:10,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/c05b854a787d419ebb3e1181141f3d56 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/c05b854a787d419ebb3e1181141f3d56 2024-11-13T18:15:10,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/a63919b5ad144b038ee248777464ee10 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/a63919b5ad144b038ee248777464ee10 2024-11-13T18:15:10,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/708c9a3101e441a394a4c9121ca20a87 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/708c9a3101e441a394a4c9121ca20a87 2024-11-13T18:15:10,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/0b9e380bf248456c9e09c5910cbb033e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/0b9e380bf248456c9e09c5910cbb033e 2024-11-13T18:15:10,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/13d48b8068a442e18a984e993aa417e9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/13d48b8068a442e18a984e993aa417e9 2024-11-13T18:15:10,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/94ff621292c84a4995bfbb14ee0da872 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/94ff621292c84a4995bfbb14ee0da872 2024-11-13T18:15:10,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/eee631051a01464982c3c84cc2279737 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/eee631051a01464982c3c84cc2279737 2024-11-13T18:15:10,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/39b092273918412da17eaada3b700e27 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/39b092273918412da17eaada3b700e27 2024-11-13T18:15:10,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1379550efd3c4748b37e1c1e8bf4fdb4 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/1379550efd3c4748b37e1c1e8bf4fdb4 2024-11-13T18:15:10,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/232c556e1cf64d8c9b75cd4be5e14348 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/232c556e1cf64d8c9b75cd4be5e14348 2024-11-13T18:15:10,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/af7b65dfbf334705af1a339dc295658a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/af7b65dfbf334705af1a339dc295658a 2024-11-13T18:15:10,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/aa82e8dc17fa44e6a549be2b6b677304 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/aa82e8dc17fa44e6a549be2b6b677304 2024-11-13T18:15:10,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/929a2776d4704199b5847f861ddba964 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/929a2776d4704199b5847f861ddba964 2024-11-13T18:15:10,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6619e09d758543be9279c771947c8318 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6619e09d758543be9279c771947c8318 2024-11-13T18:15:10,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/20c82a96f46f41d69d838668e0ea584d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/20c82a96f46f41d69d838668e0ea584d 2024-11-13T18:15:10,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c0f497255bb48b2add40f5ea2c416b5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9c0f497255bb48b2add40f5ea2c416b5 2024-11-13T18:15:10,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/054387aa4fe54d089708df09351842a5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/054387aa4fe54d089708df09351842a5 2024-11-13T18:15:10,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/88ab810b3e734534aaa7e3c07dda7a7e to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/88ab810b3e734534aaa7e3c07dda7a7e 2024-11-13T18:15:10,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9baf725c436a447e97bb7fc7fb84806f to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/9baf725c436a447e97bb7fc7fb84806f 2024-11-13T18:15:10,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/b9f0525008084b978c83f0c543e83499 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/b9f0525008084b978c83f0c543e83499 2024-11-13T18:15:10,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/51d771b5a79f47379a21136079177c69 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/51d771b5a79f47379a21136079177c69 2024-11-13T18:15:10,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/84fb410f5dd7425e941d8ac3cec9b676 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/84fb410f5dd7425e941d8ac3cec9b676 2024-11-13T18:15:10,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/680d3cf018044587a23aa2f8a279dea0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/680d3cf018044587a23aa2f8a279dea0 2024-11-13T18:15:10,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/746ac21a71ac4a438dd5038efa8176ea to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/746ac21a71ac4a438dd5038efa8176ea 2024-11-13T18:15:10,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/bf5f1562d53c47e9a937950e82f7d53b, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9a1e6b7310394471a4e196b40d8f3a68, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/091fcc07ab45498aac74feb00f767b0a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e21a1d784d0b4df09e2ecabbf4a23191, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ecc85ac0b89f451db9ee53656e56dbe1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/dcad1e9aeae74c71be58ad54b1dc409a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/d4fb4cdc4425475596fbfa19d3edd623, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ababe02cd80e46959f14ff70b73093b0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/18d7d65e6e304a0f83a118ef5f8690fc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/46a48dd3df1f4a6fbc428ad72a89cee9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/f1484d8fcd4d44e6a60443600e2f76c8, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/2832d5976e66473d932fcb47d68cf423, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/86abc88bb2154e97a2928961bc97922a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/40e7a93301474c6fa4700942553309ff, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1920c2763d8f484a80e6828cad9f8212, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/337b0a4fa987481ab3845a7814c00bd9, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9c87762607964bad8d368867c5b93ca2, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/c64705c3a025458d9ebce0c4350f99ce, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e2b0786658964be6bd30967cff811bb0, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/beb1b1f70d5d48a68a43f80e942b3f47, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/4c1d359887404dd4ae40147bb0a77dfc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/014474c1701945ea83c6768cb9a14292, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1b0c0655450e42d2ae2f3d52626d84fc, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/89b49ab0b671466b9eb5f090ce24f42a, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/a432a147123546b3993c3b31cbb0cfe5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1eb25c3143a84311b871885e37ad1583, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/48e38d06382c4ed8a27e2f77491ac7b5, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/80b4e75e9a9e4ae8a3dd1dc31183d945, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/fdf68b4bb117420fb636bad92f37f6c1, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/48f91cf5cf5b412ca7d8fe475776609c, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/246adc31b6aa45428e420192fa916300, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/7c6e73d5715a49d3ba1a2414b3ba0729, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ada383c47e8b4a8db299642a3a024076, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/5f905f3df3ec4a9492aeb93aef3ab444] to archive 2024-11-13T18:15:10,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:15:10,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/bf5f1562d53c47e9a937950e82f7d53b to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/bf5f1562d53c47e9a937950e82f7d53b 2024-11-13T18:15:10,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9a1e6b7310394471a4e196b40d8f3a68 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9a1e6b7310394471a4e196b40d8f3a68 2024-11-13T18:15:10,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/091fcc07ab45498aac74feb00f767b0a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/091fcc07ab45498aac74feb00f767b0a 2024-11-13T18:15:10,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e21a1d784d0b4df09e2ecabbf4a23191 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e21a1d784d0b4df09e2ecabbf4a23191 2024-11-13T18:15:10,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ecc85ac0b89f451db9ee53656e56dbe1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ecc85ac0b89f451db9ee53656e56dbe1 2024-11-13T18:15:10,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/dcad1e9aeae74c71be58ad54b1dc409a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/dcad1e9aeae74c71be58ad54b1dc409a 2024-11-13T18:15:10,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/d4fb4cdc4425475596fbfa19d3edd623 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/d4fb4cdc4425475596fbfa19d3edd623 2024-11-13T18:15:10,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ababe02cd80e46959f14ff70b73093b0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ababe02cd80e46959f14ff70b73093b0 2024-11-13T18:15:10,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/18d7d65e6e304a0f83a118ef5f8690fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/18d7d65e6e304a0f83a118ef5f8690fc 2024-11-13T18:15:10,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/46a48dd3df1f4a6fbc428ad72a89cee9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/46a48dd3df1f4a6fbc428ad72a89cee9 2024-11-13T18:15:10,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/f1484d8fcd4d44e6a60443600e2f76c8 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/f1484d8fcd4d44e6a60443600e2f76c8 2024-11-13T18:15:10,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/2832d5976e66473d932fcb47d68cf423 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/2832d5976e66473d932fcb47d68cf423 2024-11-13T18:15:10,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/86abc88bb2154e97a2928961bc97922a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/86abc88bb2154e97a2928961bc97922a 2024-11-13T18:15:10,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/40e7a93301474c6fa4700942553309ff to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/40e7a93301474c6fa4700942553309ff 2024-11-13T18:15:10,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1920c2763d8f484a80e6828cad9f8212 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1920c2763d8f484a80e6828cad9f8212 2024-11-13T18:15:10,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/337b0a4fa987481ab3845a7814c00bd9 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/337b0a4fa987481ab3845a7814c00bd9 2024-11-13T18:15:10,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9c87762607964bad8d368867c5b93ca2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/9c87762607964bad8d368867c5b93ca2 2024-11-13T18:15:10,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/c64705c3a025458d9ebce0c4350f99ce to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/c64705c3a025458d9ebce0c4350f99ce 2024-11-13T18:15:10,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e2b0786658964be6bd30967cff811bb0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/e2b0786658964be6bd30967cff811bb0 2024-11-13T18:15:10,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/beb1b1f70d5d48a68a43f80e942b3f47 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/beb1b1f70d5d48a68a43f80e942b3f47 2024-11-13T18:15:10,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/4c1d359887404dd4ae40147bb0a77dfc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/4c1d359887404dd4ae40147bb0a77dfc 2024-11-13T18:15:10,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/014474c1701945ea83c6768cb9a14292 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/014474c1701945ea83c6768cb9a14292 2024-11-13T18:15:10,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1b0c0655450e42d2ae2f3d52626d84fc to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1b0c0655450e42d2ae2f3d52626d84fc 2024-11-13T18:15:10,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/89b49ab0b671466b9eb5f090ce24f42a to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/89b49ab0b671466b9eb5f090ce24f42a 2024-11-13T18:15:10,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/a432a147123546b3993c3b31cbb0cfe5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/a432a147123546b3993c3b31cbb0cfe5 2024-11-13T18:15:10,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1eb25c3143a84311b871885e37ad1583 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/1eb25c3143a84311b871885e37ad1583 2024-11-13T18:15:10,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/48e38d06382c4ed8a27e2f77491ac7b5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/48e38d06382c4ed8a27e2f77491ac7b5 2024-11-13T18:15:10,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/80b4e75e9a9e4ae8a3dd1dc31183d945 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/80b4e75e9a9e4ae8a3dd1dc31183d945 2024-11-13T18:15:10,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/fdf68b4bb117420fb636bad92f37f6c1 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/fdf68b4bb117420fb636bad92f37f6c1 2024-11-13T18:15:10,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/48f91cf5cf5b412ca7d8fe475776609c to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/48f91cf5cf5b412ca7d8fe475776609c 2024-11-13T18:15:10,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/246adc31b6aa45428e420192fa916300 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/246adc31b6aa45428e420192fa916300 2024-11-13T18:15:10,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/7c6e73d5715a49d3ba1a2414b3ba0729 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/7c6e73d5715a49d3ba1a2414b3ba0729 2024-11-13T18:15:10,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ada383c47e8b4a8db299642a3a024076 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/ada383c47e8b4a8db299642a3a024076 2024-11-13T18:15:10,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/5f905f3df3ec4a9492aeb93aef3ab444 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/5f905f3df3ec4a9492aeb93aef3ab444 2024-11-13T18:15:10,833 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/recovered.edits/930.seqid, newMaxSeqId=930, maxSeqId=4 2024-11-13T18:15:10,835 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. 2024-11-13T18:15:10,835 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for a5a3ed238deaa2c83ee0dd5f472061c5: Waiting for close lock at 1731521709870Running coprocessor pre-close hooks at 1731521709870Disabling compacts and flushes for region at 1731521709870Disabling writes for close at 1731521709870Obtaining lock to block concurrent updates at 1731521709870Preparing flush snapshotting stores in a5a3ed238deaa2c83ee0dd5f472061c5 at 1731521709870Finished memstore snapshotting TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731521709870Flushing stores of TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5. at 1731521709871 (+1 ms)Flushing a5a3ed238deaa2c83ee0dd5f472061c5/A: creating writer at 1731521709871Flushing mob file a5a3ed238deaa2c83ee0dd5f472061c5/A: appending metadata at 1731521709876 (+5 ms)Flushing mob file a5a3ed238deaa2c83ee0dd5f472061c5/A: closing flushed file at 1731521709876Flushing a5a3ed238deaa2c83ee0dd5f472061c5/A: appending metadata at 1731521709902 (+26 ms)Flushing a5a3ed238deaa2c83ee0dd5f472061c5/A: closing flushed file at 1731521709902Flushing a5a3ed238deaa2c83ee0dd5f472061c5/B: creating writer at 1731521710310 (+408 ms)Flushing a5a3ed238deaa2c83ee0dd5f472061c5/B: appending metadata at 1731521710312 (+2 ms)Flushing a5a3ed238deaa2c83ee0dd5f472061c5/B: closing flushed file at 1731521710312Flushing a5a3ed238deaa2c83ee0dd5f472061c5/C: creating writer at 1731521710719 (+407 ms)Flushing a5a3ed238deaa2c83ee0dd5f472061c5/C: appending metadata at 1731521710721 (+2 ms)Flushing a5a3ed238deaa2c83ee0dd5f472061c5/C: closing flushed file at 1731521710721Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31a69a02: reopening flushed file at 1731521710727 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d170164: reopening flushed file at 1731521710730 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12b1465c: reopening flushed file at 1731521710733 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a5a3ed238deaa2c83ee0dd5f472061c5 in 866ms, sequenceid=927, compaction requested=true at 1731521710736 (+3 ms)Writing region close event to WAL at 1731521710830 (+94 ms)Running coprocessor post-close hooks at 1731521710834 (+4 ms)Closed at 1731521710835 (+1 ms) 2024-11-13T18:15:10,836 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,837 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=a5a3ed238deaa2c83ee0dd5f472061c5, regionState=CLOSED 2024-11-13T18:15:10,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 because future has completed 2024-11-13T18:15:10,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-13T18:15:10,841 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure a5a3ed238deaa2c83ee0dd5f472061c5, server=fda96cf025dc,44483,1731521526892 in 1.1220 sec 2024-11-13T18:15:10,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-13T18:15:10,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5a3ed238deaa2c83ee0dd5f472061c5, UNASSIGN in 1.1260 sec 2024-11-13T18:15:10,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-13T18:15:10,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1290 sec 2024-11-13T18:15:10,844 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521710844"}]},"ts":"1731521710844"} 2024-11-13T18:15:10,846 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:15:10,846 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:15:10,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1380 sec 2024-11-13T18:15:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T18:15:10,854 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:15:10,855 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:15:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:15:10,856 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:15:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T18:15:10,856 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:15:10,858 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,859 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C, FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/recovered.edits] 2024-11-13T18:15:10,861 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/53dcb5dd9b9542f1856cef9d080493ce to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/53dcb5dd9b9542f1856cef9d080493ce 2024-11-13T18:15:10,862 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/83a79e2dd4b0437bbf5060f227637df2 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/83a79e2dd4b0437bbf5060f227637df2 2024-11-13T18:15:10,863 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/bd9e93e0eaf24923a2731f56080cc644 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/A/bd9e93e0eaf24923a2731f56080cc644 2024-11-13T18:15:10,865 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6187e4e4173a4f08858feb9cae1475fa to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/6187e4e4173a4f08858feb9cae1475fa 2024-11-13T18:15:10,865 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/c8802bf92d914d03add2f44a87cd677d to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/c8802bf92d914d03add2f44a87cd677d 2024-11-13T18:15:10,866 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/fff00b22fffd451e93751a22a5a440c0 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/B/fff00b22fffd451e93751a22a5a440c0 2024-11-13T18:15:10,867 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/87cbb703fa7643a3add059415d950c87 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/87cbb703fa7643a3add059415d950c87 2024-11-13T18:15:10,868 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/8807917de086440ba5ab80485e5fc965 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/8807917de086440ba5ab80485e5fc965 2024-11-13T18:15:10,869 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/a1b0d3daa07749b6a05cf74df5646418 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/C/a1b0d3daa07749b6a05cf74df5646418 2024-11-13T18:15:10,871 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/recovered.edits/930.seqid to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5/recovered.edits/930.seqid 2024-11-13T18:15:10,871 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/default/TestAcidGuarantees/a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,871 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:15:10,872 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:15:10,872 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T18:15:10,874 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130250128e282d4b77860c7b1070ba8650_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130250128e282d4b77860c7b1070ba8650_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,875 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131a07662acfd24d4298878fa023e4fd95_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131a07662acfd24d4298878fa023e4fd95_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,876 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111322dadfbdb6c344a29a958d71efaec36b_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111322dadfbdb6c344a29a958d71efaec36b_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,877 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111329dceebe04ef4afb977ca5ce3941d551_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111329dceebe04ef4afb977ca5ce3941d551_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,878 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111337d89f49a81e43b3a4756a0302fb54b0_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111337d89f49a81e43b3a4756a0302fb54b0_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,879 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113402c8ca642674dc3b7f2ee3d0e941dcf_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113402c8ca642674dc3b7f2ee3d0e941dcf_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,880 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134304b4b148754b6399d1060efd4b6c81_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134304b4b148754b6399d1060efd4b6c81_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,880 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111346b00ca2fa614838bc94b608da3f6c2a_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111346b00ca2fa614838bc94b608da3f6c2a_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,881 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113661a86ef1f7340b9afe240aec9f91bae_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113661a86ef1f7340b9afe240aec9f91bae_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,882 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136794d77ea9e44cbcac6dd7b2249307b3_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136794d77ea9e44cbcac6dd7b2249307b3_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,883 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111378ad6d60a96b401485c8728b609b7d54_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111378ad6d60a96b401485c8728b609b7d54_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,884 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137955f7ba6ca44b1792e675d6c8068114_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137955f7ba6ca44b1792e675d6c8068114_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,885 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113826c69d560524e8aa35dd6fde0eef46d_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113826c69d560524e8aa35dd6fde0eef46d_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,886 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113862615869d8f4fa8b08f7d2f659a829f_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113862615869d8f4fa8b08f7d2f659a829f_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,886 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111387bb72a00f6b4efc94bab592d849c677_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111387bb72a00f6b4efc94bab592d849c677_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,887 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138f773d33e6494a2f8677463f3f7ac8e3_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138f773d33e6494a2f8677463f3f7ac8e3_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,888 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111393b436666fd048b6b8ecec8e986ce76f_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111393b436666fd048b6b8ecec8e986ce76f_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,888 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111395246000f0f643898803e8d2a856f1c5_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111395246000f0f643898803e8d2a856f1c5_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,889 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a706d5d99ea547e2a6c74dd2b56ad7ad_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a706d5d99ea547e2a6c74dd2b56ad7ad_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,890 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b341bcaaa9a54673a4c0c07504864fca_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b341bcaaa9a54673a4c0c07504864fca_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,891 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bd4b225f6a864b78aae244c3e26f0706_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bd4b225f6a864b78aae244c3e26f0706_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,891 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c1284cb8073b4caa9a2007f9cb75ee06_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c1284cb8073b4caa9a2007f9cb75ee06_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,893 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c3c1a066fc0b44859a38fcddce28b571_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c3c1a066fc0b44859a38fcddce28b571_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,893 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c5a66b52eb4b41919ec503625aecbd7a_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c5a66b52eb4b41919ec503625aecbd7a_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,894 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ca2f9791146f40fcb199742c8dde3c7e_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ca2f9791146f40fcb199742c8dde3c7e_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,895 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ddcdeefd830841cdb4ec4a9b43e01a21_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ddcdeefd830841cdb4ec4a9b43e01a21_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,896 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f407c43fd59047ae99bfc2e21a2ae06e_a5a3ed238deaa2c83ee0dd5f472061c5 to hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f407c43fd59047ae99bfc2e21a2ae06e_a5a3ed238deaa2c83ee0dd5f472061c5 2024-11-13T18:15:10,896 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:15:10,899 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:15:10,901 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:15:10,903 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:15:10,904 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:15:10,904 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:15:10,904 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731521710904"}]},"ts":"9223372036854775807"} 2024-11-13T18:15:10,906 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:15:10,906 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a5a3ed238deaa2c83ee0dd5f472061c5, NAME => 'TestAcidGuarantees,,1731521681294.a5a3ed238deaa2c83ee0dd5f472061c5.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:15:10,906 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:15:10,906 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731521710906"}]},"ts":"9223372036854775807"} 2024-11-13T18:15:10,908 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:15:10,908 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:15:10,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-13T18:15:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T18:15:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:15:10,964 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:15:10,977 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 249) - Thread LEAK? -, OpenFileDescriptor=457 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=797 (was 879), ProcessCount=11 (was 11), AvailableMemoryMB=1186 (was 278) - AvailableMemoryMB LEAK? - 2024-11-13T18:15:10,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T18:15:10,977 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:15:10,977 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:15:10,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:10,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:10,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:15:10,978 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T18:15:10,978 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1388394555, stopped=false 2024-11-13T18:15:10,978 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fda96cf025dc,37165,1731521525941 2024-11-13T18:15:10,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:15:10,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:15:10,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:10,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:10,981 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:15:10,981 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:15:10,981 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:15:10,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:10,981 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:15:10,982 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,44483,1731521526892' ***** 2024-11-13T18:15:10,982 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:15:10,983 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:15:10,983 INFO [RS:0;fda96cf025dc:44483 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:15:10,983 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:15:10,983 INFO [RS:0;fda96cf025dc:44483 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:15:10,983 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(959): stopping server fda96cf025dc,44483,1731521526892 2024-11-13T18:15:10,983 INFO [RS:0;fda96cf025dc:44483 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:15:10,983 INFO [RS:0;fda96cf025dc:44483 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fda96cf025dc:44483. 2024-11-13T18:15:10,983 DEBUG [RS:0;fda96cf025dc:44483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:15:10,983 DEBUG [RS:0;fda96cf025dc:44483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:10,984 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:15:10,984 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:15:10,984 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:15:10,985 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:15:10,985 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T18:15:10,986 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T18:15:10,986 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T18:15:10,986 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:15:10,986 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:15:10,986 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:15:10,986 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:15:10,986 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:15:10,986 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:15:10,986 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-13T18:15:11,019 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/info/2fc390d2d5174b96b435aca236c293cb is 95, key is TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed./info:/1731521557289/DeleteFamily/seqid=0 2024-11-13T18:15:11,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742638_1814 (size=6514) 2024-11-13T18:15:11,046 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:15:11,186 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:15:11,387 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:15:11,423 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/info/2fc390d2d5174b96b435aca236c293cb 2024-11-13T18:15:11,442 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/ns/d007bdf79d564e9cb55007fb0fd5fb8f is 93, key is TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed./ns:/1731521557208/DeleteFamily/seqid=0 2024-11-13T18:15:11,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742639_1815 (size=5906) 2024-11-13T18:15:11,563 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T18:15:11,563 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T18:15:11,587 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:15:11,787 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:15:11,846 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/ns/d007bdf79d564e9cb55007fb0fd5fb8f 2024-11-13T18:15:11,866 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/rep_barrier/4cd58afd974f443d8855ffbe928006af is 102, key is TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed./rep_barrier:/1731521557208/DeleteFamily/seqid=0 2024-11-13T18:15:11,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742640_1816 (size=6025) 2024-11-13T18:15:11,870 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/rep_barrier/4cd58afd974f443d8855ffbe928006af 2024-11-13T18:15:11,894 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/table/b746aa83895e4906aacd2333cdac2570 is 96, key is TestAcidGuarantees,,1731521529618.401e2f643e0c1bcf64dcec1f711ca8ed./table:/1731521557208/DeleteFamily/seqid=0 2024-11-13T18:15:11,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742641_1817 (size=5936) 2024-11-13T18:15:11,987 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:15:12,187 DEBUG [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:15:12,298 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/table/b746aa83895e4906aacd2333cdac2570 2024-11-13T18:15:12,303 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/info/2fc390d2d5174b96b435aca236c293cb as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/info/2fc390d2d5174b96b435aca236c293cb 2024-11-13T18:15:12,306 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/info/2fc390d2d5174b96b435aca236c293cb, entries=12, sequenceid=90, filesize=6.4 K 2024-11-13T18:15:12,307 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/ns/d007bdf79d564e9cb55007fb0fd5fb8f as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/ns/d007bdf79d564e9cb55007fb0fd5fb8f 2024-11-13T18:15:12,310 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/ns/d007bdf79d564e9cb55007fb0fd5fb8f, entries=8, sequenceid=90, filesize=5.8 K 2024-11-13T18:15:12,311 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/rep_barrier/4cd58afd974f443d8855ffbe928006af as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/rep_barrier/4cd58afd974f443d8855ffbe928006af 2024-11-13T18:15:12,313 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/rep_barrier/4cd58afd974f443d8855ffbe928006af, entries=6, sequenceid=90, filesize=5.9 K 2024-11-13T18:15:12,314 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/.tmp/table/b746aa83895e4906aacd2333cdac2570 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/table/b746aa83895e4906aacd2333cdac2570 2024-11-13T18:15:12,317 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/table/b746aa83895e4906aacd2333cdac2570, entries=7, sequenceid=90, filesize=5.8 K 2024-11-13T18:15:12,317 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1331ms, sequenceid=90, compaction requested=false 2024-11-13T18:15:12,323 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-13T18:15:12,324 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:15:12,324 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:15:12,324 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731521710986Running coprocessor pre-close hooks at 1731521710986Disabling compacts and flushes for region at 1731521710986Disabling writes for close at 1731521710986Obtaining lock to block concurrent updates at 1731521710986Preparing flush snapshotting stores in 1588230740 at 1731521710986Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731521710987 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731521710988 (+1 ms)Flushing 1588230740/info: creating writer at 1731521710988Flushing 1588230740/info: appending metadata at 1731521711019 (+31 ms)Flushing 1588230740/info: closing flushed file at 1731521711019Flushing 1588230740/ns: creating writer at 1731521711427 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731521711441 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731521711441Flushing 1588230740/rep_barrier: creating writer at 1731521711849 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731521711865 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731521711866 (+1 ms)Flushing 1588230740/table: creating writer at 1731521711873 (+7 ms)Flushing 1588230740/table: appending metadata at 1731521711894 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731521711894Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d722d63: reopening flushed file at 1731521712302 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e1cb044: reopening flushed file at 1731521712307 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5044c886: reopening flushed file at 1731521712310 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c2140f: reopening flushed file at 1731521712314 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1331ms, sequenceid=90, compaction requested=false at 1731521712317 (+3 ms)Writing region close event to WAL at 1731521712319 (+2 ms)Running coprocessor post-close hooks at 1731521712323 (+4 ms)Closed at 1731521712324 (+1 ms) 2024-11-13T18:15:12,324 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T18:15:12,388 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,44483,1731521526892; all regions closed. 2024-11-13T18:15:12,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741834_1010 (size=25344) 2024-11-13T18:15:12,395 DEBUG [RS:0;fda96cf025dc:44483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/oldWALs 2024-11-13T18:15:12,395 INFO [RS:0;fda96cf025dc:44483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C44483%2C1731521526892.meta:.meta(num 1731521529151) 2024-11-13T18:15:12,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741833_1009 (size=22512137) 2024-11-13T18:15:12,398 DEBUG [RS:0;fda96cf025dc:44483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/oldWALs 2024-11-13T18:15:12,398 INFO [RS:0;fda96cf025dc:44483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C44483%2C1731521526892:(num 1731521528756) 2024-11-13T18:15:12,398 DEBUG [RS:0;fda96cf025dc:44483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:12,398 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:15:12,398 INFO [RS:0;fda96cf025dc:44483 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:15:12,399 INFO [RS:0;fda96cf025dc:44483 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T18:15:12,399 INFO [RS:0;fda96cf025dc:44483 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:15:12,399 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:15:12,400 INFO [RS:0;fda96cf025dc:44483 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44483 2024-11-13T18:15:12,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,44483,1731521526892 2024-11-13T18:15:12,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:15:12,404 INFO [RS:0;fda96cf025dc:44483 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:15:12,406 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,44483,1731521526892] 2024-11-13T18:15:12,408 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,44483,1731521526892 already deleted, retry=false 2024-11-13T18:15:12,408 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fda96cf025dc,44483,1731521526892 expired; onlineServers=0 2024-11-13T18:15:12,408 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fda96cf025dc,37165,1731521525941' ***** 2024-11-13T18:15:12,408 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T18:15:12,408 INFO [M:0;fda96cf025dc:37165 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:15:12,409 INFO [M:0;fda96cf025dc:37165 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:15:12,409 DEBUG [M:0;fda96cf025dc:37165 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T18:15:12,409 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T18:15:12,409 DEBUG [M:0;fda96cf025dc:37165 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T18:15:12,409 DEBUG [master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731521528405 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731521528405,5,FailOnTimeoutGroup] 2024-11-13T18:15:12,409 DEBUG [master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731521528406 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731521528406,5,FailOnTimeoutGroup] 2024-11-13T18:15:12,409 INFO [M:0;fda96cf025dc:37165 {}] hbase.ChoreService(370): Chore service for: master/fda96cf025dc:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T18:15:12,409 INFO [M:0;fda96cf025dc:37165 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:15:12,409 DEBUG [M:0;fda96cf025dc:37165 {}] master.HMaster(1795): Stopping service threads 2024-11-13T18:15:12,409 INFO [M:0;fda96cf025dc:37165 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T18:15:12,409 INFO [M:0;fda96cf025dc:37165 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:15:12,409 ERROR [M:0;fda96cf025dc:37165 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-13T18:15:12,410 INFO [M:0;fda96cf025dc:37165 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:15:12,410 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T18:15:12,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T18:15:12,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:12,411 DEBUG [M:0;fda96cf025dc:37165 {}] zookeeper.ZKUtil(347): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T18:15:12,411 WARN [M:0;fda96cf025dc:37165 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T18:15:12,412 INFO [M:0;fda96cf025dc:37165 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/.lastflushedseqids 2024-11-13T18:15:12,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742642_1818 (size=72) 2024-11-13T18:15:12,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:15:12,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44483-0x1003a57c7cd0001, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:15:12,507 INFO [RS:0;fda96cf025dc:44483 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:15:12,508 INFO [RS:0;fda96cf025dc:44483 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,44483,1731521526892; zookeeper connection closed. 2024-11-13T18:15:12,508 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7feb65b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7feb65b 2024-11-13T18:15:12,508 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T18:15:12,824 INFO [M:0;fda96cf025dc:37165 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:15:12,824 INFO [M:0;fda96cf025dc:37165 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T18:15:12,824 DEBUG [M:0;fda96cf025dc:37165 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:15:12,824 INFO [M:0;fda96cf025dc:37165 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:15:12,824 DEBUG [M:0;fda96cf025dc:37165 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:15:12,825 DEBUG [M:0;fda96cf025dc:37165 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:15:12,825 DEBUG [M:0;fda96cf025dc:37165 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:15:12,825 INFO [M:0;fda96cf025dc:37165 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=815.99 KB heapSize=1003.26 KB 2024-11-13T18:15:12,843 DEBUG [M:0;fda96cf025dc:37165 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a033593a3dee4ac78e597f2e95915e45 is 82, key is hbase:meta,,1/info:regioninfo/1731521529273/Put/seqid=0 2024-11-13T18:15:12,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742643_1819 (size=5672) 2024-11-13T18:15:13,248 INFO [M:0;fda96cf025dc:37165 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2328 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a033593a3dee4ac78e597f2e95915e45 2024-11-13T18:15:13,279 DEBUG [M:0;fda96cf025dc:37165 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e4e6d8fbca1425880e84ff566356330 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1731521684397/Put/seqid=0 2024-11-13T18:15:13,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742644_1820 (size=45348) 2024-11-13T18:15:13,683 INFO [M:0;fda96cf025dc:37165 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=815.44 KB at sequenceid=2328 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e4e6d8fbca1425880e84ff566356330 2024-11-13T18:15:13,686 INFO [M:0;fda96cf025dc:37165 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e4e6d8fbca1425880e84ff566356330 2024-11-13T18:15:13,701 DEBUG [M:0;fda96cf025dc:37165 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8377248b4aa14f5daaf2702d680f157c is 69, key is fda96cf025dc,44483,1731521526892/rs:state/1731521528445/Put/seqid=0 2024-11-13T18:15:13,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073742645_1821 (size=5156) 2024-11-13T18:15:14,105 INFO [M:0;fda96cf025dc:37165 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2328 (bloomFilter=true), to=hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8377248b4aa14f5daaf2702d680f157c 2024-11-13T18:15:14,109 DEBUG [M:0;fda96cf025dc:37165 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a033593a3dee4ac78e597f2e95915e45 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a033593a3dee4ac78e597f2e95915e45 2024-11-13T18:15:14,113 INFO [M:0;fda96cf025dc:37165 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a033593a3dee4ac78e597f2e95915e45, entries=8, sequenceid=2328, filesize=5.5 K 2024-11-13T18:15:14,113 DEBUG [M:0;fda96cf025dc:37165 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e4e6d8fbca1425880e84ff566356330 as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e4e6d8fbca1425880e84ff566356330 2024-11-13T18:15:14,116 INFO [M:0;fda96cf025dc:37165 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e4e6d8fbca1425880e84ff566356330 2024-11-13T18:15:14,116 INFO [M:0;fda96cf025dc:37165 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e4e6d8fbca1425880e84ff566356330, entries=170, sequenceid=2328, filesize=44.3 K 2024-11-13T18:15:14,117 DEBUG [M:0;fda96cf025dc:37165 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8377248b4aa14f5daaf2702d680f157c as hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8377248b4aa14f5daaf2702d680f157c 2024-11-13T18:15:14,120 INFO [M:0;fda96cf025dc:37165 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45039/user/jenkins/test-data/3ab1e388-86b4-c6d4-a49d-e623496c22c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8377248b4aa14f5daaf2702d680f157c, entries=1, sequenceid=2328, filesize=5.0 K 2024-11-13T18:15:14,120 INFO [M:0;fda96cf025dc:37165 {}] regionserver.HRegion(3140): Finished flush of dataSize ~815.99 KB/835578, heapSize ~1002.96 KB/1027032, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1295ms, sequenceid=2328, compaction requested=false 2024-11-13T18:15:14,122 INFO [M:0;fda96cf025dc:37165 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:15:14,122 DEBUG [M:0;fda96cf025dc:37165 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731521712824Disabling compacts and flushes for region at 1731521712824Disabling writes for close at 1731521712825 (+1 ms)Obtaining lock to block concurrent updates at 1731521712825Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731521712825Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=835578, getHeapSize=1027272, getOffHeapSize=0, getCellsCount=2523 at 1731521712825Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731521712826 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731521712826Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731521712842 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731521712842Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731521713252 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731521713278 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731521713278Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731521713686 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731521713701 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731521713701Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38522b87: reopening flushed file at 1731521714109 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@766bc90: reopening flushed file at 1731521714113 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19aa2945: reopening flushed file at 1731521714116 (+3 ms)Finished flush of dataSize ~815.99 KB/835578, heapSize ~1002.96 KB/1027032, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1295ms, sequenceid=2328, compaction requested=false at 1731521714121 (+5 ms)Writing region close event to WAL at 1731521714122 (+1 ms)Closed at 1731521714122 2024-11-13T18:15:14,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38473 is added to blk_1073741830_1006 (size=988097) 2024-11-13T18:15:14,129 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:15:14,129 INFO [M:0;fda96cf025dc:37165 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T18:15:14,129 INFO [M:0;fda96cf025dc:37165 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37165 2024-11-13T18:15:14,130 INFO [M:0;fda96cf025dc:37165 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:15:14,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:15:14,233 INFO [M:0;fda96cf025dc:37165 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:15:14,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37165-0x1003a57c7cd0000, quorum=127.0.0.1:58684, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:15:14,240 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6cc47bf8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:15:14,242 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@381b1fb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:15:14,242 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:15:14,243 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b46b523{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:15:14,243 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3e3aef44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/hadoop.log.dir/,STOPPED} 2024-11-13T18:15:14,246 WARN [BP-225481686-172.17.0.3-1731521521390 heartbeating to localhost/127.0.0.1:45039 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T18:15:14,246 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T18:15:14,246 WARN [BP-225481686-172.17.0.3-1731521521390 heartbeating to localhost/127.0.0.1:45039 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-225481686-172.17.0.3-1731521521390 (Datanode Uuid b9fb38ef-faf7-4350-ae02-268f0debde47) service to localhost/127.0.0.1:45039 2024-11-13T18:15:14,246 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T18:15:14,250 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/cluster_2b3eda0b-9365-a4bb-f234-af4e8c2937bc/data/data2/current/BP-225481686-172.17.0.3-1731521521390 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:15:14,250 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/cluster_2b3eda0b-9365-a4bb-f234-af4e8c2937bc/data/data1/current/BP-225481686-172.17.0.3-1731521521390 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:15:14,251 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T18:15:14,260 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5af17d75{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:15:14,260 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@51799591{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:15:14,261 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:15:14,261 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3061ce29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:15:14,261 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72943464{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/374c5310-32d2-1a7e-2d90-b2f93fd3294a/hadoop.log.dir/,STOPPED} 2024-11-13T18:15:14,281 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T18:15:14,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down